├── deploy ├── apps │ ├── apply.sh │ ├── clean.sh │ ├── disk.yaml │ ├── cpu.yaml │ ├── net.yaml │ └── drs-scheduler.yaml └── scripts │ ├── taint.sh │ ├── env.sh │ ├── init.sh │ ├── label.sh │ └── clean_env.sh ├── drs-monitor ├── monitor.sh └── usage.py ├── drs-scheduler └── myenv │ ├── __pycache__ │ ├── K8sEnv.cpython-310.pyc │ └── __init__.cpython-310.pyc │ └── __init__.py ├── scheduler ├── OWNERS ├── framework │ ├── plugins │ │ ├── README.md │ │ ├── volumebinding │ │ │ ├── OWNERS │ │ │ ├── scorer.go │ │ │ ├── metrics │ │ │ │ └── metrics.go │ │ │ └── fake_binder.go │ │ ├── feature │ │ │ └── feature.go │ │ ├── helper │ │ │ ├── normalize_score.go │ │ │ ├── shape_score.go │ │ │ ├── normalize_score_test.go │ │ │ ├── spread_test.go │ │ │ └── spread.go │ │ ├── queuesort │ │ │ ├── priority_sort.go │ │ │ └── priority_sort_test.go │ │ ├── names │ │ │ └── names.go │ │ ├── examples │ │ │ ├── prebind │ │ │ │ └── prebind.go │ │ │ ├── multipoint │ │ │ │ └── multipoint.go │ │ │ └── stateful │ │ │ │ └── stateful.go │ │ ├── noderesources │ │ │ ├── least_allocated.go │ │ │ ├── test_util.go │ │ │ ├── most_allocated.go │ │ │ ├── requested_to_capacity_ratio.go │ │ │ └── balanced_allocation.go │ │ ├── defaultbinder │ │ │ ├── default_binder.go │ │ │ └── default_binder_test.go │ │ ├── nodename │ │ │ ├── node_name_test.go │ │ │ └── node_name.go │ │ ├── nodeunschedulable │ │ │ ├── node_unschedulable_test.go │ │ │ └── node_unschedulable.go │ │ ├── dqn │ │ │ └── dqn.go │ │ ├── testing │ │ │ └── testing.go │ │ ├── nodevolumelimits │ │ │ └── utils.go │ │ ├── selectorspread │ │ │ └── selector_spread_perf_test.go │ │ ├── podtopologyspread │ │ │ └── common.go │ │ ├── registry.go │ │ └── nodeports │ │ │ └── node_ports.go │ ├── listers.go │ ├── parallelize │ │ ├── parallelism_test.go │ │ ├── error_channel_test.go │ │ ├── error_channel.go │ │ └── parallelism.go │ ├── cycle_state_test.go │ ├── extender.go │ ├── runtime │ │ ├── metrics_recorder.go │ │ ├── registry.go │ │ └── waiting_pods_map.go │ └── cycle_state.go ├── apis │ └── config │ │ ├── OWNERS │ │ ├── doc.go │ │ ├── v1beta2 │ │ ├── zz_generated.deepcopy.go │ │ ├── doc.go │ │ ├── register.go │ │ ├── zz_generated.defaults.go │ │ └── conversion.go │ │ ├── v1beta3 │ │ ├── zz_generated.deepcopy.go │ │ ├── doc.go │ │ ├── register.go │ │ ├── zz_generated.defaults.go │ │ └── conversion.go │ │ ├── types_test.go │ │ ├── latest │ │ └── latest.go │ │ ├── scheme │ │ └── scheme.go │ │ ├── register.go │ │ └── testing │ │ └── config.go ├── internal │ ├── cache │ │ ├── debugger │ │ │ ├── signal_windows.go │ │ │ ├── signal.go │ │ │ ├── debugger.go │ │ │ ├── dumper.go │ │ │ └── comparer.go │ │ ├── fake │ │ │ └── fake_cache.go │ │ ├── node_tree.go │ │ └── snapshot_test.go │ └── queue │ │ └── testing.go ├── util │ ├── clock.go │ ├── pod_resources.go │ └── pod_resources_test.go ├── metrics │ ├── profile_metrics.go │ ├── metric_recorder.go │ └── metric_recorder_test.go └── profile │ └── profile.go └── README.md /deploy/apps/apply.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubectl apply -f $1 4 | -------------------------------------------------------------------------------- /drs-monitor/monitor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python3 -u monitor.py > node.log 2>&1 -------------------------------------------------------------------------------- /deploy/apps/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubectl delete -f my-scheduler.yaml 4 | docker container prune 5 | -------------------------------------------------------------------------------- /deploy/scripts/taint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubectl taint nodes --all node-role.kubernetes.io/master- 4 | -------------------------------------------------------------------------------- /drs-scheduler/myenv/__pycache__/K8sEnv.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JolyonJian/DRS/HEAD/drs-scheduler/myenv/__pycache__/K8sEnv.cpython-310.pyc -------------------------------------------------------------------------------- /drs-scheduler/myenv/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JolyonJian/DRS/HEAD/drs-scheduler/myenv/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /drs-scheduler/myenv/__init__.py: -------------------------------------------------------------------------------- 1 | from gym.envs.registration import register 2 | 3 | register( 4 | id = "k8s-v0", 5 | entry_point = "myenv.K8sEnv:K8sEnv" 6 | ) -------------------------------------------------------------------------------- /deploy/scripts/env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mkdir -p $HOME/.kube 4 | sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 5 | sudo chown $(id -u):$(id -g) $HOME/.kube/config 6 | -------------------------------------------------------------------------------- /scheduler/OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | approvers: 4 | - sig-scheduling-maintainers 5 | reviewers: 6 | - sig-scheduling 7 | labels: 8 | - sig/scheduling 9 | -------------------------------------------------------------------------------- /deploy/scripts/init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubeadm init \ 4 | --image-repository registry.aliyuncs.com/google_containers \ 5 | --pod-network-cidr=10.244.0.0/16 \ 6 | --kubernetes-version v1.23.4 7 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/README.md: -------------------------------------------------------------------------------- 1 | # Scheduler Framework Plugins 2 | 3 | Moved [here](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-scheduling/scheduler_framework_plugins.md). 4 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/volumebinding/OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | approvers: 4 | - sig-storage-approvers 5 | - cofyc 6 | reviewers: 7 | - sig-storage-reviewers 8 | - cofyc 9 | - lichuqiang 10 | labels: 11 | - sig/storage 12 | -------------------------------------------------------------------------------- /deploy/scripts/label.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubectl label nodes node1 node-role.kubernetes.io/worker= 4 | kubectl label nodes node2 node-role.kubernetes.io/worker= 5 | kubectl label nodes node3 node-role.kubernetes.io/worker= 6 | kubectl label nodes node4 node-role.kubernetes.io/worker= 7 | -------------------------------------------------------------------------------- /scheduler/apis/config/OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | approvers: 4 | - api-approvers 5 | reviewers: 6 | - api-reviewers 7 | - sig-scheduling-api-reviewers 8 | - sig-scheduling-api-approvers 9 | labels: 10 | - kind/api-change 11 | - sig/scheduling 12 | -------------------------------------------------------------------------------- /deploy/scripts/clean_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo rm -rf $HOME/.kube/* 4 | sudo rm -rf /var/lib/cni/ 5 | sudo rm -rf /var/lib/kubelet/* 6 | sudo rm -rf /etc/kubernetes/ 7 | sudo rm -rf /etc/cni/ 8 | sudo ifconfig cni0 down 9 | sudo ifconfig flannel.1 down 10 | sudo ip link delete cni0 11 | sudo ip link delete flannel.1 12 | -------------------------------------------------------------------------------- /deploy/apps/disk.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: app-disk 5 | spec: 6 | schedulerName: my-scheduler 7 | nodeName: node1 8 | containers: 9 | - name: app-disk 10 | image: jolyonjian/apps:io-1.0 11 | command: ["/workspace/run.sh"] 12 | resources: 13 | limits: 14 | cpu: 500m 15 | requests: 16 | cpu: 500m 17 | restartPolicy: OnFailure 18 | -------------------------------------------------------------------------------- /deploy/apps/cpu.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: app-video 5 | spec: 6 | schedulerName: my-scheduler 7 | nodeName: node1 8 | containers: 9 | - name: app-video 10 | image: jolyonjian/apps:cpu-1.0 11 | command: ["/tmp/workdir/run.sh"] 12 | resources: 13 | limits: 14 | cpu: 500m 15 | requests: 16 | cpu: 500m 17 | restartPolicy: OnFailure 18 | -------------------------------------------------------------------------------- /deploy/apps/net.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: app-net 5 | spec: 6 | schedulerName: my-scheduler 7 | nodeName: node1 8 | containers: 9 | - name: app-net 10 | image: jolyonjian/apps:net-1.0 11 | command: ["/workspace/run.sh"] 12 | resources: 13 | limits: 14 | cpu: "500m" 15 | requests: 16 | cpu: "500m" 17 | restartPolicy: OnFailure 18 | -------------------------------------------------------------------------------- /scheduler/apis/config/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // +k8s:deepcopy-gen=package 18 | // +groupName=kubescheduler.config.k8s.io 19 | 20 | package config // import "k8s.io/kubernetes/pkg/scheduler/apis/config" 21 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta2/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Code generated by deepcopy-gen. DO NOT EDIT. 21 | 22 | package v1beta2 23 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta3/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Code generated by deepcopy-gen. DO NOT EDIT. 21 | 22 | package v1beta3 23 | -------------------------------------------------------------------------------- /scheduler/internal/cache/debugger/signal_windows.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package debugger 18 | 19 | import "os" 20 | 21 | // compareSignal is the signal to trigger cache compare. For windows, 22 | // it's SIGINT. 23 | var compareSignal = os.Interrupt 24 | -------------------------------------------------------------------------------- /scheduler/internal/cache/debugger/signal.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | /* 5 | Copyright 2018 The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | package debugger 21 | 22 | import "syscall" 23 | 24 | // compareSignal is the signal to trigger cache compare. For non-windows 25 | // environment it's SIGUSR2. 26 | var compareSignal = syscall.SIGUSR2 27 | -------------------------------------------------------------------------------- /scheduler/util/clock.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "time" 21 | ) 22 | 23 | // Clock provides an interface for getting the current time 24 | type Clock interface { 25 | Now() time.Time 26 | } 27 | 28 | // RealClock implements a clock using time 29 | type RealClock struct{} 30 | 31 | // Now returns the current time with time.Now 32 | func (RealClock) Now() time.Time { 33 | return time.Now() 34 | } 35 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta2/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // +k8s:deepcopy-gen=package 18 | // +k8s:conversion-gen=k8s.io/kubernetes/pkg/scheduler/apis/config 19 | // +k8s:conversion-gen-external-types=k8s.io/kube-scheduler/config/v1beta2 20 | // +k8s:defaulter-gen=TypeMeta 21 | // +k8s:defaulter-gen-input=k8s.io/kube-scheduler/config/v1beta2 22 | // +groupName=kubescheduler.config.k8s.io 23 | 24 | package v1beta2 // import "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2" 25 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta3/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // +k8s:deepcopy-gen=package 18 | // +k8s:conversion-gen=k8s.io/kubernetes/pkg/scheduler/apis/config 19 | // +k8s:conversion-gen-external-types=k8s.io/kube-scheduler/config/v1beta3 20 | // +k8s:defaulter-gen=TypeMeta 21 | // +k8s:defaulter-gen-input=k8s.io/kube-scheduler/config/v1beta3 22 | // +groupName=kubescheduler.config.k8s.io 23 | 24 | package v1beta3 // import "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3" 25 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/feature/feature.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package feature 18 | 19 | // Features carries feature gate values used by various plugins. 20 | // This struct allows us to break the dependency of the plugins on 21 | // the internal k8s features pkg. 22 | type Features struct { 23 | EnablePodAffinityNamespaceSelector bool 24 | EnablePodDisruptionBudget bool 25 | EnablePodOverhead bool 26 | EnableReadWriteOncePod bool 27 | EnableVolumeCapacityPriority bool 28 | EnableCSIStorageCapacity bool 29 | } 30 | -------------------------------------------------------------------------------- /scheduler/framework/listers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package framework 18 | 19 | // NodeInfoLister interface represents anything that can list/get NodeInfo objects from node name. 20 | type NodeInfoLister interface { 21 | // Returns the list of NodeInfos. 22 | List() ([]*NodeInfo, error) 23 | // Returns the list of NodeInfos of nodes with pods with affinity terms. 24 | HavePodsWithAffinityList() ([]*NodeInfo, error) 25 | // Returns the list of NodeInfos of nodes with pods with required anti-affinity terms. 26 | HavePodsWithRequiredAntiAffinityList() ([]*NodeInfo, error) 27 | // Returns the NodeInfo of the given node name. 28 | Get(nodeName string) (*NodeInfo, error) 29 | } 30 | 31 | // SharedLister groups scheduler-specific listers. 32 | type SharedLister interface { 33 | NodeInfos() NodeInfoLister 34 | } 35 | -------------------------------------------------------------------------------- /scheduler/framework/parallelize/parallelism_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package parallelize 18 | 19 | import ( 20 | "fmt" 21 | "testing" 22 | ) 23 | 24 | func TestChunkSize(t *testing.T) { 25 | tests := []struct { 26 | input int 27 | wantOutput int 28 | }{ 29 | { 30 | input: 32, 31 | wantOutput: 3, 32 | }, 33 | { 34 | input: 16, 35 | wantOutput: 2, 36 | }, 37 | { 38 | input: 1, 39 | wantOutput: 1, 40 | }, 41 | { 42 | input: 0, 43 | wantOutput: 1, 44 | }, 45 | } 46 | 47 | for _, test := range tests { 48 | t.Run(fmt.Sprintf("%d", test.input), func(t *testing.T) { 49 | if chunkSizeFor(test.input, DefaultParallelism) != test.wantOutput { 50 | t.Errorf("Expected: %d, got: %d", test.wantOutput, chunkSizeFor(test.input, DefaultParallelism)) 51 | } 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /scheduler/framework/parallelize/error_channel_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package parallelize 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "testing" 23 | ) 24 | 25 | func TestErrorChannel(t *testing.T) { 26 | errCh := NewErrorChannel() 27 | 28 | if actualErr := errCh.ReceiveError(); actualErr != nil { 29 | t.Errorf("expect nil from err channel, but got %v", actualErr) 30 | } 31 | 32 | err := errors.New("unknown error") 33 | errCh.SendError(err) 34 | if actualErr := errCh.ReceiveError(); actualErr != err { 35 | t.Errorf("expect %v from err channel, but got %v", err, actualErr) 36 | } 37 | 38 | ctx, cancel := context.WithCancel(context.Background()) 39 | errCh.SendErrorWithCancel(err, cancel) 40 | if actualErr := errCh.ReceiveError(); actualErr != err { 41 | t.Errorf("expect %v from err channel, but got %v", err, actualErr) 42 | } 43 | 44 | if ctxErr := ctx.Err(); ctxErr != context.Canceled { 45 | t.Errorf("expect context canceled, but got %v", ctxErr) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/helper/normalize_score.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helper 18 | 19 | import ( 20 | "k8s.io/kubernetes/pkg/scheduler/framework" 21 | ) 22 | 23 | // DefaultNormalizeScore generates a Normalize Score function that can normalize the 24 | // scores to [0, maxPriority]. If reverse is set to true, it reverses the scores by 25 | // subtracting it from maxPriority. 26 | func DefaultNormalizeScore(maxPriority int64, reverse bool, scores framework.NodeScoreList) *framework.Status { 27 | var maxCount int64 28 | for i := range scores { 29 | if scores[i].Score > maxCount { 30 | maxCount = scores[i].Score 31 | } 32 | } 33 | 34 | if maxCount == 0 { 35 | if reverse { 36 | for i := range scores { 37 | scores[i].Score = maxPriority 38 | } 39 | } 40 | return nil 41 | } 42 | 43 | for i := range scores { 44 | score := scores[i].Score 45 | 46 | score = maxPriority * score / maxCount 47 | if reverse { 48 | score = maxPriority - score 49 | } 50 | 51 | scores[i].Score = score 52 | } 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /scheduler/apis/config/types_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/google/go-cmp/cmp" 23 | ) 24 | 25 | func TestPluginsNames(t *testing.T) { 26 | tests := []struct { 27 | name string 28 | plugins *Plugins 29 | want []string 30 | }{ 31 | { 32 | name: "empty", 33 | }, 34 | { 35 | name: "with duplicates", 36 | plugins: &Plugins{ 37 | Filter: PluginSet{ 38 | Enabled: []Plugin{ 39 | {Name: "CustomFilter"}, 40 | }, 41 | }, 42 | PreFilter: PluginSet{ 43 | Enabled: []Plugin{ 44 | {Name: "CustomFilter"}, 45 | }, 46 | }, 47 | Score: PluginSet{ 48 | Enabled: []Plugin{ 49 | {Name: "CustomScore"}, 50 | }, 51 | }, 52 | }, 53 | want: []string{"CustomFilter", "CustomScore"}, 54 | }, 55 | } 56 | for _, test := range tests { 57 | t.Run(test.name, func(t *testing.T) { 58 | if d := cmp.Diff(test.want, test.plugins.Names()); d != "" { 59 | t.Fatalf("plugins mismatch (-want +got):\n%s", d) 60 | } 61 | }) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta2/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta2 18 | 19 | import ( 20 | "k8s.io/kube-scheduler/config/v1beta2" 21 | ) 22 | 23 | // GroupName is the group name used in this package 24 | const GroupName = v1beta2.GroupName 25 | 26 | // SchemeGroupVersion is group version used to register these objects 27 | var SchemeGroupVersion = v1beta2.SchemeGroupVersion 28 | 29 | var ( 30 | // localSchemeBuilder extends the SchemeBuilder instance with the external types. In this package, 31 | // defaulting and conversion init funcs are registered as well. 32 | localSchemeBuilder = &v1beta2.SchemeBuilder 33 | // AddToScheme is a global function that registers this API group & version to a scheme 34 | AddToScheme = localSchemeBuilder.AddToScheme 35 | ) 36 | 37 | func init() { 38 | // We only register manually written functions here. The registration of the 39 | // generated functions takes place in the generated files. The separation 40 | // makes the code compile even when the generated files are missing. 41 | localSchemeBuilder.Register(addDefaultingFuncs) 42 | } 43 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta3/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta3 18 | 19 | import ( 20 | "k8s.io/kube-scheduler/config/v1beta3" 21 | ) 22 | 23 | // GroupName is the group name used in this package 24 | const GroupName = v1beta3.GroupName 25 | 26 | // SchemeGroupVersion is group version used to register these objects 27 | var SchemeGroupVersion = v1beta3.SchemeGroupVersion 28 | 29 | var ( 30 | // localSchemeBuilder extends the SchemeBuilder instance with the external types. In this package, 31 | // defaulting and conversion init funcs are registered as well. 32 | localSchemeBuilder = &v1beta3.SchemeBuilder 33 | // AddToScheme is a global function that registers this API group & version to a scheme 34 | AddToScheme = localSchemeBuilder.AddToScheme 35 | ) 36 | 37 | func init() { 38 | // We only register manually written functions here. The registration of the 39 | // generated functions takes place in the generated files. The separation 40 | // makes the code compile even when the generated files are missing. 41 | localSchemeBuilder.Register(addDefaultingFuncs) 42 | } 43 | -------------------------------------------------------------------------------- /scheduler/internal/queue/testing.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package queue 18 | 19 | import ( 20 | "context" 21 | 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/client-go/informers" 24 | "k8s.io/client-go/kubernetes/fake" 25 | "k8s.io/kubernetes/pkg/scheduler/framework" 26 | ) 27 | 28 | // NewTestQueueWithObjects creates a priority queue with an informer factory 29 | // populated with the provided objects. 30 | func NewTestQueueWithObjects( 31 | ctx context.Context, 32 | lessFn framework.LessFunc, 33 | objs []runtime.Object, 34 | opts ...Option, 35 | ) *PriorityQueue { 36 | informerFactory := informers.NewSharedInformerFactory(fake.NewSimpleClientset(objs...), 0) 37 | pq := NewPriorityQueue(lessFn, informerFactory, opts...) 38 | informerFactory.Start(ctx.Done()) 39 | informerFactory.WaitForCacheSync(ctx.Done()) 40 | return pq 41 | } 42 | 43 | // NewTestQueue creates a priority queue with an empty informer factory. 44 | func NewTestQueue(ctx context.Context, lessFn framework.LessFunc, opts ...Option) *PriorityQueue { 45 | return NewTestQueueWithObjects(ctx, lessFn, nil, opts...) 46 | } 47 | -------------------------------------------------------------------------------- /scheduler/apis/config/latest/latest.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package latest 18 | 19 | import ( 20 | "k8s.io/component-base/config/v1alpha1" 21 | "k8s.io/kube-scheduler/config/v1beta3" 22 | "k8s.io/kubernetes/pkg/scheduler/apis/config" 23 | "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" 24 | ) 25 | 26 | // Default creates a default configuration of the latest versioned type. 27 | // This function needs to be updated whenever we bump the scheduler's component config version. 28 | func Default() (*config.KubeSchedulerConfiguration, error) { 29 | versionedCfg := v1beta3.KubeSchedulerConfiguration{} 30 | versionedCfg.DebuggingConfiguration = *v1alpha1.NewRecommendedDebuggingConfiguration() 31 | 32 | scheme.Scheme.Default(&versionedCfg) 33 | cfg := config.KubeSchedulerConfiguration{} 34 | if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil { 35 | return nil, err 36 | } 37 | // We don't set this field in pkg/scheduler/apis/config/{version}/conversion.go 38 | // because the field will be cleared later by API machinery during 39 | // conversion. See KubeSchedulerConfiguration internal type definition for 40 | // more details. 41 | cfg.TypeMeta.APIVersion = v1beta3.SchemeGroupVersion.String() 42 | return &cfg, nil 43 | } 44 | -------------------------------------------------------------------------------- /scheduler/apis/config/scheme/scheme.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package scheme 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime" 21 | "k8s.io/apimachinery/pkg/runtime/serializer" 22 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 23 | config "k8s.io/kubernetes/pkg/scheduler/apis/config" 24 | configv1beta2 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2" 25 | configv1beta3 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3" 26 | ) 27 | 28 | var ( 29 | // Scheme is the runtime.Scheme to which all kubescheduler api types are registered. 30 | Scheme = runtime.NewScheme() 31 | 32 | // Codecs provides access to encoding and decoding for the scheme. 33 | Codecs = serializer.NewCodecFactory(Scheme, serializer.EnableStrict) 34 | ) 35 | 36 | func init() { 37 | AddToScheme(Scheme) 38 | } 39 | 40 | // AddToScheme builds the kubescheduler scheme using all known versions of the kubescheduler api. 41 | func AddToScheme(scheme *runtime.Scheme) { 42 | utilruntime.Must(config.AddToScheme(scheme)) 43 | utilruntime.Must(configv1beta2.AddToScheme(scheme)) 44 | utilruntime.Must(configv1beta3.AddToScheme(scheme)) 45 | utilruntime.Must(scheme.SetVersionPriority( 46 | configv1beta3.SchemeGroupVersion, 47 | configv1beta2.SchemeGroupVersion)) 48 | } 49 | -------------------------------------------------------------------------------- /scheduler/apis/config/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime" 21 | "k8s.io/apimachinery/pkg/runtime/schema" 22 | ) 23 | 24 | // GroupName is the group name used in this package 25 | const GroupName = "kubescheduler.config.k8s.io" 26 | 27 | // SchemeGroupVersion is group version used to register these objects 28 | var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} 29 | 30 | var ( 31 | // SchemeBuilder is the scheme builder with scheme init functions to run for this API package 32 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 33 | // AddToScheme is a global function that registers this API group & version to a scheme 34 | AddToScheme = SchemeBuilder.AddToScheme 35 | ) 36 | 37 | // addKnownTypes registers known types to the given scheme 38 | func addKnownTypes(scheme *runtime.Scheme) error { 39 | scheme.AddKnownTypes(SchemeGroupVersion, 40 | &KubeSchedulerConfiguration{}, 41 | &DefaultPreemptionArgs{}, 42 | &InterPodAffinityArgs{}, 43 | &NodeResourcesFitArgs{}, 44 | &PodTopologySpreadArgs{}, 45 | &VolumeBindingArgs{}, 46 | &NodeResourcesBalancedAllocationArgs{}, 47 | &NodeAffinityArgs{}, 48 | ) 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /scheduler/framework/parallelize/error_channel.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package parallelize 18 | 19 | import "context" 20 | 21 | // ErrorChannel supports non-blocking send and receive operation to capture error. 22 | // A maximum of one error is kept in the channel and the rest of the errors sent 23 | // are ignored, unless the existing error is received and the channel becomes empty 24 | // again. 25 | type ErrorChannel struct { 26 | errCh chan error 27 | } 28 | 29 | // SendError sends an error without blocking the sender. 30 | func (e *ErrorChannel) SendError(err error) { 31 | select { 32 | case e.errCh <- err: 33 | default: 34 | } 35 | } 36 | 37 | // SendErrorWithCancel sends an error without blocking the sender and calls 38 | // cancel function. 39 | func (e *ErrorChannel) SendErrorWithCancel(err error, cancel context.CancelFunc) { 40 | e.SendError(err) 41 | cancel() 42 | } 43 | 44 | // ReceiveError receives an error from channel without blocking on the receiver. 45 | func (e *ErrorChannel) ReceiveError() error { 46 | select { 47 | case err := <-e.errCh: 48 | return err 49 | default: 50 | return nil 51 | } 52 | } 53 | 54 | // NewErrorChannel returns a new ErrorChannel. 55 | func NewErrorChannel() *ErrorChannel { 56 | return &ErrorChannel{ 57 | errCh: make(chan error, 1), 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /scheduler/framework/parallelize/parallelism.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package parallelize 18 | 19 | import ( 20 | "context" 21 | "math" 22 | 23 | "k8s.io/client-go/util/workqueue" 24 | ) 25 | 26 | // DefaultParallelism is the default parallelism used in scheduler. 27 | const DefaultParallelism int = 16 28 | 29 | // Parallelizer holds the parallelism for scheduler. 30 | type Parallelizer struct { 31 | parallelism int 32 | } 33 | 34 | // NewParallelizer returns an object holding the parallelism. 35 | func NewParallelizer(p int) Parallelizer { 36 | return Parallelizer{parallelism: p} 37 | } 38 | 39 | // chunkSizeFor returns a chunk size for the given number of items to use for 40 | // parallel work. The size aims to produce good CPU utilization. 41 | // returns max(1, min(sqrt(n), n/Parallelism)) 42 | func chunkSizeFor(n, parallelism int) int { 43 | s := int(math.Sqrt(float64(n))) 44 | 45 | if r := n/parallelism + 1; s > r { 46 | s = r 47 | } else if s < 1 { 48 | s = 1 49 | } 50 | return s 51 | } 52 | 53 | // Until is a wrapper around workqueue.ParallelizeUntil to use in scheduling algorithms. 54 | func (p Parallelizer) Until(ctx context.Context, pieces int, doWorkPiece workqueue.DoWorkPieceFunc) { 55 | workqueue.ParallelizeUntil(ctx, p.parallelism, pieces, doWorkPiece, workqueue.WithChunkSize(chunkSizeFor(pieces, p.parallelism))) 56 | } 57 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/queuesort/priority_sort.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package queuesort 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime" 21 | corev1helpers "k8s.io/component-helpers/scheduling/corev1" 22 | "k8s.io/kubernetes/pkg/scheduler/framework" 23 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" 24 | ) 25 | 26 | // Name is the name of the plugin used in the plugin registry and configurations. 27 | const Name = names.PrioritySort 28 | 29 | // PrioritySort is a plugin that implements Priority based sorting. 30 | type PrioritySort struct{} 31 | 32 | var _ framework.QueueSortPlugin = &PrioritySort{} 33 | 34 | // Name returns name of the plugin. 35 | func (pl *PrioritySort) Name() string { 36 | return Name 37 | } 38 | 39 | // Less is the function used by the activeQ heap algorithm to sort pods. 40 | // It sorts pods based on their priority. When priorities are equal, it uses 41 | // PodQueueInfo.timestamp. 42 | func (pl *PrioritySort) Less(pInfo1, pInfo2 *framework.QueuedPodInfo) bool { 43 | p1 := corev1helpers.PodPriority(pInfo1.Pod) 44 | p2 := corev1helpers.PodPriority(pInfo2.Pod) 45 | return (p1 > p2) || (p1 == p2 && pInfo1.Timestamp.Before(pInfo2.Timestamp)) 46 | } 47 | 48 | // New initializes a new plugin and returns it. 49 | func New(_ runtime.Object, handle framework.Handle) (framework.Plugin, error) { 50 | return &PrioritySort{}, nil 51 | } 52 | -------------------------------------------------------------------------------- /scheduler/metrics/profile_metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | // This file contains helpers for metrics that are associated to a profile. 20 | 21 | var ( 22 | scheduledResult = "scheduled" 23 | unschedulableResult = "unschedulable" 24 | errorResult = "error" 25 | ) 26 | 27 | // PodScheduled can records a successful scheduling attempt and the duration 28 | // since `start`. 29 | func PodScheduled(profile string, duration float64) { 30 | observeScheduleAttemptAndLatency(scheduledResult, profile, duration) 31 | } 32 | 33 | // PodUnschedulable can records a scheduling attempt for an unschedulable pod 34 | // and the duration since `start`. 35 | func PodUnschedulable(profile string, duration float64) { 36 | observeScheduleAttemptAndLatency(unschedulableResult, profile, duration) 37 | } 38 | 39 | // PodScheduleError can records a scheduling attempt that had an error and the 40 | // duration since `start`. 41 | func PodScheduleError(profile string, duration float64) { 42 | observeScheduleAttemptAndLatency(errorResult, profile, duration) 43 | } 44 | 45 | func observeScheduleAttemptAndLatency(result, profile string, duration float64) { 46 | e2eSchedulingLatency.WithLabelValues(result, profile).Observe(duration) 47 | schedulingLatency.WithLabelValues(result, profile).Observe(duration) 48 | scheduleAttempts.WithLabelValues(result, profile).Inc() 49 | } 50 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/names/names.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package names 18 | 19 | const ( 20 | PrioritySort = "PrioritySort" 21 | DefaultBinder = "DefaultBinder" 22 | DefaultPreemption = "DefaultPreemption" 23 | ImageLocality = "ImageLocality" 24 | InterPodAffinity = "InterPodAffinity" 25 | NodeAffinity = "NodeAffinity" 26 | NodeName = "NodeName" 27 | NodePorts = "NodePorts" 28 | NodeResourcesBalancedAllocation = "NodeResourcesBalancedAllocation" 29 | NodeResourcesFit = "NodeResourcesFit" 30 | NodeUnschedulable = "NodeUnschedulable" 31 | NodeVolumeLimits = "NodeVolumeLimits" 32 | AzureDiskLimits = "AzureDiskLimits" 33 | CinderLimits = "CinderLimits" 34 | EBSLimits = "EBSLimits" 35 | GCEPDLimits = "GCEPDLimits" 36 | PodTopologySpread = "PodTopologySpread" 37 | SelectorSpread = "SelectorSpread" 38 | ServiceAffinity = "ServiceAffinity" 39 | TaintToleration = "TaintToleration" 40 | VolumeBinding = "VolumeBinding" 41 | VolumeRestrictions = "VolumeRestrictions" 42 | VolumeZone = "VolumeZone" 43 | ) 44 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/helper/shape_score.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helper 18 | 19 | // FunctionShape represents a collection of FunctionShapePoint. 20 | type FunctionShape []FunctionShapePoint 21 | 22 | // FunctionShapePoint represents a shape point. 23 | type FunctionShapePoint struct { 24 | // Utilization is function argument. 25 | Utilization int64 26 | // Score is function value. 27 | Score int64 28 | } 29 | 30 | // BuildBrokenLinearFunction creates a function which is built using linear segments. Segments are defined via shape array. 31 | // Shape[i].Utilization slice represents points on "Utilization" axis where different segments meet. 32 | // Shape[i].Score represents function values at meeting points. 33 | // 34 | // function f(p) is defined as: 35 | // shape[0].Score for p < shape[0].Utilization 36 | // shape[n-1].Score for p > shape[n-1].Utilization 37 | // and linear between points (p < shape[i].Utilization) 38 | func BuildBrokenLinearFunction(shape FunctionShape) func(int64) int64 { 39 | return func(p int64) int64 { 40 | for i := 0; i < len(shape); i++ { 41 | if p <= int64(shape[i].Utilization) { 42 | if i == 0 { 43 | return shape[0].Score 44 | } 45 | return shape[i-1].Score + (shape[i].Score-shape[i-1].Score)*(p-shape[i-1].Utilization)/(shape[i].Utilization-shape[i-1].Utilization) 46 | } 47 | } 48 | return shape[len(shape)-1].Score 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /scheduler/apis/config/testing/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package testing 18 | 19 | import ( 20 | "testing" 21 | 22 | "k8s.io/component-base/config/v1alpha1" 23 | "k8s.io/kube-scheduler/config/v1beta2" 24 | "k8s.io/kube-scheduler/config/v1beta3" 25 | "k8s.io/kubernetes/pkg/scheduler/apis/config" 26 | "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" 27 | ) 28 | 29 | // V1beta2ToInternalWithDefaults creates a v1beta2 default configuration. 30 | func V1beta2ToInternalWithDefaults(t *testing.T, versionedCfg v1beta2.KubeSchedulerConfiguration) *config.KubeSchedulerConfiguration { 31 | versionedCfg.DebuggingConfiguration = *v1alpha1.NewRecommendedDebuggingConfiguration() 32 | 33 | scheme.Scheme.Default(&versionedCfg) 34 | cfg := config.KubeSchedulerConfiguration{} 35 | if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil { 36 | t.Fatal(err) 37 | } 38 | return &cfg 39 | } 40 | 41 | // V1beta3ToInternalWithDefaults creates a v1beta3 default configuration. 42 | func V1beta3ToInternalWithDefaults(t *testing.T, versionedCfg v1beta3.KubeSchedulerConfiguration) *config.KubeSchedulerConfiguration { 43 | versionedCfg.DebuggingConfiguration = *v1alpha1.NewRecommendedDebuggingConfiguration() 44 | 45 | scheme.Scheme.Default(&versionedCfg) 46 | cfg := config.KubeSchedulerConfiguration{} 47 | if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil { 48 | t.Fatal(err) 49 | } 50 | return &cfg 51 | } 52 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/volumebinding/scorer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package volumebinding 18 | 19 | import ( 20 | "math" 21 | 22 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" 23 | ) 24 | 25 | // classResourceMap holds a map of storage class to resource. 26 | type classResourceMap map[string]*StorageResource 27 | 28 | // volumeCapacityScorer calculates the score based on class storage resource information. 29 | type volumeCapacityScorer func(classResourceMap) int64 30 | 31 | // buildScorerFunction builds volumeCapacityScorer from the scoring function shape. 32 | func buildScorerFunction(scoringFunctionShape helper.FunctionShape) volumeCapacityScorer { 33 | rawScoringFunction := helper.BuildBrokenLinearFunction(scoringFunctionShape) 34 | f := func(requested, capacity int64) int64 { 35 | if capacity == 0 || requested > capacity { 36 | return rawScoringFunction(maxUtilization) 37 | } 38 | 39 | return rawScoringFunction(requested * maxUtilization / capacity) 40 | } 41 | return func(classResources classResourceMap) int64 { 42 | var nodeScore int64 43 | // in alpha stage, all classes have the same weight 44 | weightSum := len(classResources) 45 | if weightSum == 0 { 46 | return 0 47 | } 48 | for _, resource := range classResources { 49 | classScore := f(resource.Requested, resource.Capacity) 50 | nodeScore += classScore 51 | } 52 | return int64(math.Round(float64(nodeScore) / float64(weightSum))) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/examples/prebind/prebind.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package prebind 18 | 19 | import ( 20 | "context" 21 | v1 "k8s.io/api/core/v1" 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/kubernetes/pkg/scheduler/framework" 24 | ) 25 | 26 | // StatelessPreBindExample is an example of a simple plugin that has no state 27 | // and implements only one hook for prebind. 28 | type StatelessPreBindExample struct{} 29 | 30 | var _ framework.PreBindPlugin = StatelessPreBindExample{} 31 | 32 | // Name is the name of the plugin used in Registry and configurations. 33 | const Name = "stateless-prebind-plugin-example" 34 | 35 | // Name returns name of the plugin. It is used in logs, etc. 36 | func (sr StatelessPreBindExample) Name() string { 37 | return Name 38 | } 39 | 40 | // PreBind is the functions invoked by the framework at "prebind" extension point. 41 | func (sr StatelessPreBindExample) PreBind(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) *framework.Status { 42 | if pod == nil { 43 | return framework.NewStatus(framework.Error, "pod cannot be nil") 44 | } 45 | if pod.Namespace != "foo" { 46 | return framework.NewStatus(framework.Unschedulable, "only pods from 'foo' namespace are allowed") 47 | } 48 | return nil 49 | } 50 | 51 | // New initializes a new plugin and returns it. 52 | func New(_ *runtime.Unknown, _ framework.Handle) (framework.Plugin, error) { 53 | return &StatelessPreBindExample{}, nil 54 | } 55 | -------------------------------------------------------------------------------- /scheduler/framework/cycle_state_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package framework 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | type fakeData struct { 24 | data string 25 | } 26 | 27 | func (f *fakeData) Clone() StateData { 28 | copy := &fakeData{ 29 | data: f.data, 30 | } 31 | return copy 32 | } 33 | 34 | func TestCycleStateClone(t *testing.T) { 35 | var key StateKey = "key" 36 | data1 := "value1" 37 | data2 := "value2" 38 | 39 | state := NewCycleState() 40 | originalValue := &fakeData{ 41 | data: data1, 42 | } 43 | state.Write(key, originalValue) 44 | stateCopy := state.Clone() 45 | 46 | valueCopy, err := stateCopy.Read(key) 47 | if err != nil { 48 | t.Errorf("failed to read copied value: %v", err) 49 | } 50 | if v, ok := valueCopy.(*fakeData); ok && v.data != data1 { 51 | t.Errorf("clone failed, got %q, expected %q", v.data, data1) 52 | } 53 | 54 | originalValue.data = data2 55 | original, err := state.Read(key) 56 | if err != nil { 57 | t.Errorf("failed to read original value: %v", err) 58 | } 59 | if v, ok := original.(*fakeData); ok && v.data != data2 { 60 | t.Errorf("original value should change, got %q, expected %q", v.data, data2) 61 | } 62 | 63 | if v, ok := valueCopy.(*fakeData); ok && v.data != data1 { 64 | t.Errorf("cloned copy should not change, got %q, expected %q", v.data, data1) 65 | } 66 | } 67 | 68 | func TestCycleStateCloneNil(t *testing.T) { 69 | var state *CycleState 70 | stateCopy := state.Clone() 71 | if stateCopy != nil { 72 | t.Errorf("clone expected to be nil") 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /scheduler/internal/cache/debugger/debugger.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package debugger 18 | 19 | import ( 20 | "os" 21 | "os/signal" 22 | 23 | corelisters "k8s.io/client-go/listers/core/v1" 24 | internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" 25 | internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" 26 | ) 27 | 28 | // CacheDebugger provides ways to check and write cache information for debugging. 29 | type CacheDebugger struct { 30 | Comparer CacheComparer 31 | Dumper CacheDumper 32 | } 33 | 34 | // New creates a CacheDebugger. 35 | func New( 36 | nodeLister corelisters.NodeLister, 37 | podLister corelisters.PodLister, 38 | cache internalcache.Cache, 39 | podQueue internalqueue.SchedulingQueue, 40 | ) *CacheDebugger { 41 | return &CacheDebugger{ 42 | Comparer: CacheComparer{ 43 | NodeLister: nodeLister, 44 | PodLister: podLister, 45 | Cache: cache, 46 | PodQueue: podQueue, 47 | }, 48 | Dumper: CacheDumper{ 49 | cache: cache, 50 | podQueue: podQueue, 51 | }, 52 | } 53 | } 54 | 55 | // ListenForSignal starts a goroutine that will trigger the CacheDebugger's 56 | // behavior when the process receives SIGINT (Windows) or SIGUSER2 (non-Windows). 57 | func (d *CacheDebugger) ListenForSignal(stopCh <-chan struct{}) { 58 | ch := make(chan os.Signal, 1) 59 | signal.Notify(ch, compareSignal) 60 | 61 | go func() { 62 | for { 63 | select { 64 | case <-stopCh: 65 | return 66 | case <-ch: 67 | d.Comparer.Compare() 68 | d.Dumper.DumpAll() 69 | } 70 | } 71 | }() 72 | } 73 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/volumebinding/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "k8s.io/component-base/metrics" 21 | "k8s.io/component-base/metrics/legacyregistry" 22 | ) 23 | 24 | // VolumeSchedulerSubsystem - subsystem name used by scheduler 25 | const VolumeSchedulerSubsystem = "scheduler_volume" 26 | 27 | var ( 28 | // VolumeBindingRequestSchedulerBinderCache tracks the number of volume binder cache operations. 29 | VolumeBindingRequestSchedulerBinderCache = metrics.NewCounterVec( 30 | &metrics.CounterOpts{ 31 | Subsystem: VolumeSchedulerSubsystem, 32 | Name: "binder_cache_requests_total", 33 | Help: "Total number for request volume binding cache", 34 | StabilityLevel: metrics.ALPHA, 35 | }, 36 | []string{"operation"}, 37 | ) 38 | // VolumeSchedulingStageFailed tracks the number of failed volume scheduling operations. 39 | VolumeSchedulingStageFailed = metrics.NewCounterVec( 40 | &metrics.CounterOpts{ 41 | Subsystem: VolumeSchedulerSubsystem, 42 | Name: "scheduling_stage_error_total", 43 | Help: "Volume scheduling stage error count", 44 | StabilityLevel: metrics.ALPHA, 45 | }, 46 | []string{"operation"}, 47 | ) 48 | ) 49 | 50 | // RegisterVolumeSchedulingMetrics is used for scheduler, because the volume binding cache is a library 51 | // used by scheduler process. 52 | func RegisterVolumeSchedulingMetrics() { 53 | legacyregistry.MustRegister(VolumeBindingRequestSchedulerBinderCache) 54 | legacyregistry.MustRegister(VolumeSchedulingStageFailed) 55 | } 56 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/noderesources/least_allocated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package noderesources 18 | 19 | import ( 20 | "k8s.io/kubernetes/pkg/scheduler/framework" 21 | ) 22 | 23 | // leastResourceScorer favors nodes with fewer requested resources. 24 | // It calculates the percentage of memory and CPU requested by pods scheduled on the node, and 25 | // prioritizes based on the minimum of the average of the fraction of requested to capacity. 26 | // 27 | // Details: 28 | // (cpu((capacity-sum(requested))*MaxNodeScore/capacity) + memory((capacity-sum(requested))*MaxNodeScore/capacity))/weightSum 29 | func leastResourceScorer(resToWeightMap resourceToWeightMap) func(resourceToValueMap, resourceToValueMap) int64 { 30 | return func(requested, allocable resourceToValueMap) int64 { 31 | var nodeScore, weightSum int64 32 | for resource := range requested { 33 | weight := resToWeightMap[resource] 34 | resourceScore := leastRequestedScore(requested[resource], allocable[resource]) 35 | nodeScore += resourceScore * weight 36 | weightSum += weight 37 | } 38 | if weightSum == 0 { 39 | return 0 40 | } 41 | return nodeScore / weightSum 42 | } 43 | } 44 | 45 | // The unused capacity is calculated on a scale of 0-MaxNodeScore 46 | // 0 being the lowest priority and `MaxNodeScore` being the highest. 47 | // The more unused resources the higher the score is. 48 | func leastRequestedScore(requested, capacity int64) int64 { 49 | if capacity == 0 { 50 | return 0 51 | } 52 | if requested > capacity { 53 | return 0 54 | } 55 | 56 | return ((capacity - requested) * int64(framework.MaxNodeScore)) / capacity 57 | } 58 | -------------------------------------------------------------------------------- /scheduler/metrics/metric_recorder.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "k8s.io/component-base/metrics" 21 | ) 22 | 23 | // MetricRecorder represents a metric recorder which takes action when the 24 | // metric Inc(), Dec() and Clear() 25 | type MetricRecorder interface { 26 | Inc() 27 | Dec() 28 | Clear() 29 | } 30 | 31 | var _ MetricRecorder = &PendingPodsRecorder{} 32 | 33 | // PendingPodsRecorder is an implementation of MetricRecorder 34 | type PendingPodsRecorder struct { 35 | recorder metrics.GaugeMetric 36 | } 37 | 38 | // NewActivePodsRecorder returns ActivePods in a Prometheus metric fashion 39 | func NewActivePodsRecorder() *PendingPodsRecorder { 40 | return &PendingPodsRecorder{ 41 | recorder: ActivePods(), 42 | } 43 | } 44 | 45 | // NewUnschedulablePodsRecorder returns UnschedulablePods in a Prometheus metric fashion 46 | func NewUnschedulablePodsRecorder() *PendingPodsRecorder { 47 | return &PendingPodsRecorder{ 48 | recorder: UnschedulablePods(), 49 | } 50 | } 51 | 52 | // NewBackoffPodsRecorder returns BackoffPods in a Prometheus metric fashion 53 | func NewBackoffPodsRecorder() *PendingPodsRecorder { 54 | return &PendingPodsRecorder{ 55 | recorder: BackoffPods(), 56 | } 57 | } 58 | 59 | // Inc increases a metric counter by 1, in an atomic way 60 | func (r *PendingPodsRecorder) Inc() { 61 | r.recorder.Inc() 62 | } 63 | 64 | // Dec decreases a metric counter by 1, in an atomic way 65 | func (r *PendingPodsRecorder) Dec() { 66 | r.recorder.Dec() 67 | } 68 | 69 | // Clear set a metric counter to 0, in an atomic way 70 | func (r *PendingPodsRecorder) Clear() { 71 | r.recorder.Set(float64(0)) 72 | } 73 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/defaultbinder/default_binder.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package defaultbinder 18 | 19 | import ( 20 | "context" 21 | 22 | v1 "k8s.io/api/core/v1" 23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/klog/v2" 26 | "k8s.io/kubernetes/pkg/scheduler/framework" 27 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" 28 | ) 29 | 30 | // Name of the plugin used in the plugin registry and configurations. 31 | const Name = names.DefaultBinder 32 | 33 | // DefaultBinder binds pods to nodes using a k8s client. 34 | type DefaultBinder struct { 35 | handle framework.Handle 36 | } 37 | 38 | var _ framework.BindPlugin = &DefaultBinder{} 39 | 40 | // New creates a DefaultBinder. 41 | func New(_ runtime.Object, handle framework.Handle) (framework.Plugin, error) { 42 | return &DefaultBinder{handle: handle}, nil 43 | } 44 | 45 | // Name returns the name of the plugin. 46 | func (b DefaultBinder) Name() string { 47 | return Name 48 | } 49 | 50 | // Bind binds pods to nodes using the k8s client. 51 | func (b DefaultBinder) Bind(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) *framework.Status { 52 | klog.V(3).InfoS("Attempting to bind pod to node", "pod", klog.KObj(p), "node", nodeName) 53 | binding := &v1.Binding{ 54 | ObjectMeta: metav1.ObjectMeta{Namespace: p.Namespace, Name: p.Name, UID: p.UID}, 55 | Target: v1.ObjectReference{Kind: "Node", Name: nodeName}, 56 | } 57 | err := b.handle.ClientSet().CoreV1().Pods(binding.Namespace).Bind(ctx, binding, metav1.CreateOptions{}) 58 | if err != nil { 59 | return framework.AsStatus(err) 60 | } 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/nodename/node_name_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nodename 18 | 19 | import ( 20 | "context" 21 | "reflect" 22 | "testing" 23 | 24 | v1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/kubernetes/pkg/scheduler/framework" 27 | ) 28 | 29 | func TestNodeName(t *testing.T) { 30 | tests := []struct { 31 | pod *v1.Pod 32 | node *v1.Node 33 | name string 34 | wantStatus *framework.Status 35 | }{ 36 | { 37 | pod: &v1.Pod{}, 38 | node: &v1.Node{}, 39 | name: "no host specified", 40 | }, 41 | { 42 | pod: &v1.Pod{ 43 | Spec: v1.PodSpec{ 44 | NodeName: "foo", 45 | }, 46 | }, 47 | node: &v1.Node{ 48 | ObjectMeta: metav1.ObjectMeta{ 49 | Name: "foo", 50 | }, 51 | }, 52 | name: "host matches", 53 | }, 54 | { 55 | pod: &v1.Pod{ 56 | Spec: v1.PodSpec{ 57 | NodeName: "bar", 58 | }, 59 | }, 60 | node: &v1.Node{ 61 | ObjectMeta: metav1.ObjectMeta{ 62 | Name: "foo", 63 | }, 64 | }, 65 | name: "host doesn't match", 66 | wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReason), 67 | }, 68 | } 69 | 70 | for _, test := range tests { 71 | t.Run(test.name, func(t *testing.T) { 72 | nodeInfo := framework.NewNodeInfo() 73 | nodeInfo.SetNode(test.node) 74 | 75 | p, _ := New(nil, nil) 76 | gotStatus := p.(framework.FilterPlugin).Filter(context.Background(), nil, test.pod, nodeInfo) 77 | if !reflect.DeepEqual(gotStatus, test.wantStatus) { 78 | t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) 79 | } 80 | }) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/noderesources/test_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package noderesources 18 | 19 | import ( 20 | "github.com/google/go-cmp/cmp/cmpopts" 21 | v1 "k8s.io/api/core/v1" 22 | "k8s.io/apimachinery/pkg/api/resource" 23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | "k8s.io/apimachinery/pkg/util/validation/field" 25 | ) 26 | 27 | var ( 28 | ignoreBadValueDetail = cmpopts.IgnoreFields(field.Error{}, "BadValue", "Detail") 29 | ) 30 | 31 | func makeNode(node string, milliCPU, memory int64) *v1.Node { 32 | return &v1.Node{ 33 | ObjectMeta: metav1.ObjectMeta{Name: node}, 34 | Status: v1.NodeStatus{ 35 | Capacity: v1.ResourceList{ 36 | v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), 37 | v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), 38 | }, 39 | Allocatable: v1.ResourceList{ 40 | v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), 41 | v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), 42 | }, 43 | }, 44 | } 45 | } 46 | 47 | func makeNodeWithExtendedResource(node string, milliCPU, memory int64, extendedResource map[string]int64) *v1.Node { 48 | resourceList := make(map[v1.ResourceName]resource.Quantity) 49 | for res, quantity := range extendedResource { 50 | resourceList[v1.ResourceName(res)] = *resource.NewQuantity(quantity, resource.DecimalSI) 51 | } 52 | resourceList[v1.ResourceCPU] = *resource.NewMilliQuantity(milliCPU, resource.DecimalSI) 53 | resourceList[v1.ResourceMemory] = *resource.NewQuantity(memory, resource.BinarySI) 54 | return &v1.Node{ 55 | ObjectMeta: metav1.ObjectMeta{Name: node}, 56 | Status: v1.NodeStatus{ 57 | Capacity: resourceList, 58 | Allocatable: resourceList, 59 | }, 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/noderesources/most_allocated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package noderesources 18 | 19 | import ( 20 | "k8s.io/kubernetes/pkg/scheduler/framework" 21 | ) 22 | 23 | // mostResourceScorer favors nodes with most requested resources. 24 | // It calculates the percentage of memory and CPU requested by pods scheduled on the node, and prioritizes 25 | // based on the maximum of the average of the fraction of requested to capacity. 26 | // 27 | // Details: 28 | // (cpu(MaxNodeScore * sum(requested) / capacity) + memory(MaxNodeScore * sum(requested) / capacity)) / weightSum 29 | func mostResourceScorer(resToWeightMap resourceToWeightMap) func(requested, allocable resourceToValueMap) int64 { 30 | return func(requested, allocable resourceToValueMap) int64 { 31 | var nodeScore, weightSum int64 32 | for resource := range requested { 33 | weight := resToWeightMap[resource] 34 | resourceScore := mostRequestedScore(requested[resource], allocable[resource]) 35 | nodeScore += resourceScore * weight 36 | weightSum += weight 37 | } 38 | if weightSum == 0 { 39 | return 0 40 | } 41 | return nodeScore / weightSum 42 | } 43 | } 44 | 45 | // The used capacity is calculated on a scale of 0-MaxNodeScore (MaxNodeScore is 46 | // constant with value set to 100). 47 | // 0 being the lowest priority and 100 being the highest. 48 | // The more resources are used the higher the score is. This function 49 | // is almost a reversed version of noderesources.leastRequestedScore. 50 | func mostRequestedScore(requested, capacity int64) int64 { 51 | if capacity == 0 { 52 | return 0 53 | } 54 | if requested > capacity { 55 | // `requested` might be greater than `capacity` because pods with no 56 | // requests get minimum values. 57 | requested = capacity 58 | } 59 | 60 | return (requested * framework.MaxNodeScore) / capacity 61 | } 62 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/nodeunschedulable/node_unschedulable_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nodeunschedulable 18 | 19 | import ( 20 | "context" 21 | "reflect" 22 | "testing" 23 | 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/kubernetes/pkg/scheduler/framework" 26 | ) 27 | 28 | func TestNodeUnschedulable(t *testing.T) { 29 | testCases := []struct { 30 | name string 31 | pod *v1.Pod 32 | node *v1.Node 33 | wantStatus *framework.Status 34 | }{ 35 | { 36 | name: "Does not schedule pod to unschedulable node (node.Spec.Unschedulable==true)", 37 | pod: &v1.Pod{}, 38 | node: &v1.Node{ 39 | Spec: v1.NodeSpec{ 40 | Unschedulable: true, 41 | }, 42 | }, 43 | wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonUnschedulable), 44 | }, 45 | { 46 | name: "Schedule pod to normal node", 47 | pod: &v1.Pod{}, 48 | node: &v1.Node{ 49 | Spec: v1.NodeSpec{ 50 | Unschedulable: false, 51 | }, 52 | }, 53 | }, 54 | { 55 | name: "Schedule pod with toleration to unschedulable node (node.Spec.Unschedulable==true)", 56 | pod: &v1.Pod{ 57 | Spec: v1.PodSpec{ 58 | Tolerations: []v1.Toleration{ 59 | { 60 | Key: v1.TaintNodeUnschedulable, 61 | Effect: v1.TaintEffectNoSchedule, 62 | }, 63 | }, 64 | }, 65 | }, 66 | node: &v1.Node{ 67 | Spec: v1.NodeSpec{ 68 | Unschedulable: true, 69 | }, 70 | }, 71 | }, 72 | } 73 | 74 | for _, test := range testCases { 75 | nodeInfo := framework.NewNodeInfo() 76 | nodeInfo.SetNode(test.node) 77 | 78 | p, _ := New(nil, nil) 79 | gotStatus := p.(framework.FilterPlugin).Filter(context.Background(), nil, test.pod, nodeInfo) 80 | if !reflect.DeepEqual(gotStatus, test.wantStatus) { 81 | t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/helper/normalize_score_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helper 18 | 19 | import ( 20 | "fmt" 21 | "testing" 22 | 23 | "github.com/google/go-cmp/cmp" 24 | "k8s.io/kubernetes/pkg/scheduler/framework" 25 | ) 26 | 27 | func TestDefaultNormalizeScore(t *testing.T) { 28 | tests := []struct { 29 | reverse bool 30 | scores []int64 31 | expectedScores []int64 32 | }{ 33 | { 34 | scores: []int64{1, 2, 3, 4}, 35 | expectedScores: []int64{25, 50, 75, 100}, 36 | }, 37 | { 38 | reverse: true, 39 | scores: []int64{1, 2, 3, 4}, 40 | expectedScores: []int64{75, 50, 25, 0}, 41 | }, 42 | { 43 | scores: []int64{1000, 10, 20, 30}, 44 | expectedScores: []int64{100, 1, 2, 3}, 45 | }, 46 | { 47 | reverse: true, 48 | scores: []int64{1000, 10, 20, 30}, 49 | expectedScores: []int64{0, 99, 98, 97}, 50 | }, 51 | { 52 | scores: []int64{1, 1, 1, 1}, 53 | expectedScores: []int64{100, 100, 100, 100}, 54 | }, 55 | { 56 | scores: []int64{1000, 1, 1, 1}, 57 | expectedScores: []int64{100, 0, 0, 0}, 58 | }, 59 | { 60 | reverse: true, 61 | scores: []int64{0, 1, 1, 1}, 62 | expectedScores: []int64{100, 0, 0, 0}, 63 | }, 64 | } 65 | 66 | for i, test := range tests { 67 | t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { 68 | scores := framework.NodeScoreList{} 69 | for _, score := range test.scores { 70 | scores = append(scores, framework.NodeScore{Score: score}) 71 | } 72 | 73 | expectedScores := framework.NodeScoreList{} 74 | for _, score := range test.expectedScores { 75 | expectedScores = append(expectedScores, framework.NodeScore{Score: score}) 76 | } 77 | 78 | DefaultNormalizeScore(framework.MaxNodeScore, test.reverse, scores) 79 | if diff := cmp.Diff(expectedScores, scores); diff != "" { 80 | t.Errorf("Unexpected scores (-want, +got):\n%s", diff) 81 | } 82 | }) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/nodename/node_name.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nodename 18 | 19 | import ( 20 | "context" 21 | 22 | v1 "k8s.io/api/core/v1" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | "k8s.io/kubernetes/pkg/scheduler/framework" 25 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" 26 | ) 27 | 28 | // NodeName is a plugin that checks if a pod spec node name matches the current node. 29 | type NodeName struct{} 30 | 31 | var _ framework.FilterPlugin = &NodeName{} 32 | var _ framework.EnqueueExtensions = &NodeName{} 33 | 34 | const ( 35 | // Name is the name of the plugin used in the plugin registry and configurations. 36 | Name = names.NodeName 37 | 38 | // ErrReason returned when node name doesn't match. 39 | ErrReason = "node(s) didn't match the requested node name" 40 | ) 41 | 42 | // EventsToRegister returns the possible events that may make a Pod 43 | // failed by this plugin schedulable. 44 | func (pl *NodeName) EventsToRegister() []framework.ClusterEvent { 45 | return []framework.ClusterEvent{ 46 | {Resource: framework.Node, ActionType: framework.Add}, 47 | } 48 | } 49 | 50 | // Name returns name of the plugin. It is used in logs, etc. 51 | func (pl *NodeName) Name() string { 52 | return Name 53 | } 54 | 55 | // Filter invoked at the filter extension point. 56 | func (pl *NodeName) Filter(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 57 | if nodeInfo.Node() == nil { 58 | return framework.NewStatus(framework.Error, "node not found") 59 | } 60 | if !Fits(pod, nodeInfo) { 61 | return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReason) 62 | } 63 | return nil 64 | } 65 | 66 | // Fits actually checks if the pod fits the node. 67 | func Fits(pod *v1.Pod, nodeInfo *framework.NodeInfo) bool { 68 | return len(pod.Spec.NodeName) == 0 || pod.Spec.NodeName == nodeInfo.Node().Name 69 | } 70 | 71 | // New initializes a new plugin and returns it. 72 | func New(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) { 73 | return &NodeName{}, nil 74 | } 75 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/volumebinding/fake_binder.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package volumebinding 18 | 19 | import v1 "k8s.io/api/core/v1" 20 | 21 | // FakeVolumeBinderConfig holds configurations for fake volume binder. 22 | type FakeVolumeBinderConfig struct { 23 | AllBound bool 24 | FindReasons ConflictReasons 25 | FindErr error 26 | AssumeErr error 27 | BindErr error 28 | } 29 | 30 | // NewFakeVolumeBinder sets up all the caches needed for the scheduler to make 31 | // topology-aware volume binding decisions. 32 | func NewFakeVolumeBinder(config *FakeVolumeBinderConfig) *FakeVolumeBinder { 33 | return &FakeVolumeBinder{ 34 | config: config, 35 | } 36 | } 37 | 38 | // FakeVolumeBinder represents a fake volume binder for testing. 39 | type FakeVolumeBinder struct { 40 | config *FakeVolumeBinderConfig 41 | AssumeCalled bool 42 | BindCalled bool 43 | } 44 | 45 | // GetPodVolumes implements SchedulerVolumeBinder.GetPodVolumes. 46 | func (b *FakeVolumeBinder) GetPodVolumes(pod *v1.Pod) (boundClaims, unboundClaimsDelayBinding, unboundClaimsImmediate []*v1.PersistentVolumeClaim, err error) { 47 | return nil, nil, nil, nil 48 | } 49 | 50 | // FindPodVolumes implements SchedulerVolumeBinder.FindPodVolumes. 51 | func (b *FakeVolumeBinder) FindPodVolumes(pod *v1.Pod, _, _ []*v1.PersistentVolumeClaim, node *v1.Node) (podVolumes *PodVolumes, reasons ConflictReasons, err error) { 52 | return nil, b.config.FindReasons, b.config.FindErr 53 | } 54 | 55 | // AssumePodVolumes implements SchedulerVolumeBinder.AssumePodVolumes. 56 | func (b *FakeVolumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string, podVolumes *PodVolumes) (bool, error) { 57 | b.AssumeCalled = true 58 | return b.config.AllBound, b.config.AssumeErr 59 | } 60 | 61 | // RevertAssumedPodVolumes implements SchedulerVolumeBinder.RevertAssumedPodVolumes 62 | func (b *FakeVolumeBinder) RevertAssumedPodVolumes(_ *PodVolumes) {} 63 | 64 | // BindPodVolumes implements SchedulerVolumeBinder.BindPodVolumes. 65 | func (b *FakeVolumeBinder) BindPodVolumes(assumedPod *v1.Pod, podVolumes *PodVolumes) error { 66 | b.BindCalled = true 67 | return b.config.BindErr 68 | } 69 | -------------------------------------------------------------------------------- /scheduler/metrics/metric_recorder_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "sync" 21 | "sync/atomic" 22 | "testing" 23 | ) 24 | 25 | var _ MetricRecorder = &fakePodsRecorder{} 26 | 27 | type fakePodsRecorder struct { 28 | counter int64 29 | } 30 | 31 | func (r *fakePodsRecorder) Inc() { 32 | atomic.AddInt64(&r.counter, 1) 33 | } 34 | 35 | func (r *fakePodsRecorder) Dec() { 36 | atomic.AddInt64(&r.counter, -1) 37 | } 38 | 39 | func (r *fakePodsRecorder) Clear() { 40 | atomic.StoreInt64(&r.counter, 0) 41 | } 42 | 43 | func TestInc(t *testing.T) { 44 | fakeRecorder := fakePodsRecorder{} 45 | var wg sync.WaitGroup 46 | loops := 100 47 | wg.Add(loops) 48 | for i := 0; i < loops; i++ { 49 | go func() { 50 | fakeRecorder.Inc() 51 | wg.Done() 52 | }() 53 | } 54 | wg.Wait() 55 | if fakeRecorder.counter != int64(loops) { 56 | t.Errorf("Expected %v, got %v", loops, fakeRecorder.counter) 57 | } 58 | } 59 | 60 | func TestDec(t *testing.T) { 61 | fakeRecorder := fakePodsRecorder{counter: 100} 62 | var wg sync.WaitGroup 63 | loops := 100 64 | wg.Add(loops) 65 | for i := 0; i < loops; i++ { 66 | go func() { 67 | fakeRecorder.Dec() 68 | wg.Done() 69 | }() 70 | } 71 | wg.Wait() 72 | if fakeRecorder.counter != int64(0) { 73 | t.Errorf("Expected %v, got %v", loops, fakeRecorder.counter) 74 | } 75 | } 76 | 77 | func TestClear(t *testing.T) { 78 | fakeRecorder := fakePodsRecorder{} 79 | var wg sync.WaitGroup 80 | incLoops, decLoops := 100, 80 81 | wg.Add(incLoops + decLoops) 82 | for i := 0; i < incLoops; i++ { 83 | go func() { 84 | fakeRecorder.Inc() 85 | wg.Done() 86 | }() 87 | } 88 | for i := 0; i < decLoops; i++ { 89 | go func() { 90 | fakeRecorder.Dec() 91 | wg.Done() 92 | }() 93 | } 94 | wg.Wait() 95 | if fakeRecorder.counter != int64(incLoops-decLoops) { 96 | t.Errorf("Expected %v, got %v", incLoops-decLoops, fakeRecorder.counter) 97 | } 98 | // verify Clear() works 99 | fakeRecorder.Clear() 100 | if fakeRecorder.counter != int64(0) { 101 | t.Errorf("Expected %v, got %v", 0, fakeRecorder.counter) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/dqn/dqn.go: -------------------------------------------------------------------------------- 1 | package dqn 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/url" 9 | 10 | v1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | framework "k8s.io/kubernetes/pkg/scheduler/framework" 13 | ) 14 | 15 | const ( 16 | // Name is the name of the plugin used in the plugin registry and configurations. 17 | Name = "dqn-plugin" 18 | // ErrReason returned when node name doesn't match. 19 | ErrReason = "This node is not the result given by the RL scheudler" 20 | ) 21 | 22 | type DQNPlugin struct { 23 | choose string 24 | handle framework.Handle 25 | } 26 | 27 | // var _ framework.PreFilterPlugin = DQNPlugin{} 28 | var _ framework.FilterPlugin = DQNPlugin{} 29 | 30 | func (dp DQNPlugin) Name() string { 31 | return Name 32 | } 33 | 34 | // func (dp DQNPlugin) PreFilter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod) *framework.Status { 35 | // fmt.Printf("[INFO] Prefilter pod: %v\n\n", pod.Name) 36 | // return framework.NewStatus(framework.Success, "") 37 | // } 38 | 39 | // func (dp DQNPlugin) PreFilterExtensions() framework.PreFilterExtensions { 40 | // return nil 41 | // } 42 | 43 | func (dp DQNPlugin) Filter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 44 | fmt.Printf("[INFO] Filter pod: %v\n\n", pod.Name) 45 | node := nodeInfo.Node() 46 | if node == nil { 47 | return framework.NewStatus(framework.Error, "node not find") 48 | } 49 | 50 | // get result from rl algorithm 51 | if dp.choose == "" { 52 | schedulerurl := "http://192.168.1.113:1234/choose" 53 | urlValues := url.Values{} 54 | urlValues.Add("podname", pod.Name) 55 | resp, err := http.PostForm(schedulerurl, urlValues) 56 | if err != nil { 57 | fmt.Printf("[ERROR] Get choose from %v failed!\n\n", schedulerurl) 58 | fmt.Printf("[INFO] RL scheduling failed, all node will pass the filter!\n\n") 59 | dp.choose = "error" 60 | } 61 | body, _ := ioutil.ReadAll(resp.Body) 62 | dp.choose = string(body) 63 | } 64 | 65 | if dp.choose == "error" { 66 | fmt.Printf("[INFO] Filter pod: %v, node: %v\n\n", pod.Name, nodeInfo.Node().Name) 67 | return framework.NewStatus(framework.Success, "") 68 | } else { 69 | if node.Name == dp.choose { 70 | fmt.Printf("[INFO] Filter pod: %v, node: %v\n\n", pod.Name, nodeInfo.Node().Name) 71 | return framework.NewStatus(framework.Success, "") 72 | } else { 73 | fmt.Printf("[INFO] Filter pod: %v, node: %v, reason: %v\n\n", pod.Name, nodeInfo.Node().Name, ErrReason) 74 | return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReason) 75 | } 76 | } 77 | } 78 | 79 | func New(configuration runtime.Object, h framework.Handle) (framework.Plugin, error) { 80 | return &DQNPlugin{choose: "", handle: h}, nil 81 | } 82 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/testing/testing.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package testing 18 | 19 | import ( 20 | "context" 21 | "testing" 22 | 23 | "k8s.io/api/core/v1" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/client-go/informers" 27 | "k8s.io/client-go/kubernetes/fake" 28 | "k8s.io/kubernetes/pkg/scheduler/framework" 29 | frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 30 | ) 31 | 32 | // SetupPluginWithInformers creates a plugin using a framework handle that includes 33 | // the provided sharedLister and a SharedInformerFactory with the provided objects. 34 | // The function also creates an empty namespace (since most tests creates pods with 35 | // empty namespace), and start informer factory. 36 | func SetupPluginWithInformers( 37 | ctx context.Context, 38 | tb testing.TB, 39 | pf frameworkruntime.PluginFactory, 40 | config runtime.Object, 41 | sharedLister framework.SharedLister, 42 | objs []runtime.Object, 43 | ) framework.Plugin { 44 | objs = append([]runtime.Object{&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ""}}}, objs...) 45 | informerFactory := informers.NewSharedInformerFactory(fake.NewSimpleClientset(objs...), 0) 46 | fh, err := frameworkruntime.NewFramework(nil, nil, 47 | frameworkruntime.WithSnapshotSharedLister(sharedLister), 48 | frameworkruntime.WithInformerFactory(informerFactory)) 49 | if err != nil { 50 | tb.Fatalf("Failed creating framework runtime: %v", err) 51 | } 52 | p, err := pf(config, fh) 53 | if err != nil { 54 | tb.Fatal(err) 55 | } 56 | informerFactory.Start(ctx.Done()) 57 | informerFactory.WaitForCacheSync(ctx.Done()) 58 | return p 59 | } 60 | 61 | // SetupPlugin creates a plugin using a framework handle that includes 62 | // the provided sharedLister. 63 | func SetupPlugin( 64 | tb testing.TB, 65 | pf frameworkruntime.PluginFactory, 66 | config runtime.Object, 67 | sharedLister framework.SharedLister, 68 | ) framework.Plugin { 69 | fh, err := frameworkruntime.NewFramework(nil, nil, 70 | frameworkruntime.WithSnapshotSharedLister(sharedLister)) 71 | if err != nil { 72 | tb.Fatalf("Failed creating framework runtime: %v", err) 73 | } 74 | p, err := pf(config, fh) 75 | if err != nil { 76 | tb.Fatal(err) 77 | } 78 | return p 79 | } 80 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/defaultbinder/default_binder_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package defaultbinder 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "testing" 23 | 24 | "github.com/google/go-cmp/cmp" 25 | v1 "k8s.io/api/core/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/client-go/kubernetes/fake" 29 | clienttesting "k8s.io/client-go/testing" 30 | frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 31 | ) 32 | 33 | func TestDefaultBinder(t *testing.T) { 34 | testPod := &v1.Pod{ 35 | ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "ns"}, 36 | } 37 | testNode := "foohost.kubernetes.mydomain.com" 38 | tests := []struct { 39 | name string 40 | injectErr error 41 | wantBinding *v1.Binding 42 | }{ 43 | { 44 | name: "successful", 45 | wantBinding: &v1.Binding{ 46 | ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "foo"}, 47 | Target: v1.ObjectReference{Kind: "Node", Name: testNode}, 48 | }, 49 | }, { 50 | name: "binding error", 51 | injectErr: errors.New("binding error"), 52 | }, 53 | } 54 | for _, tt := range tests { 55 | t.Run(tt.name, func(t *testing.T) { 56 | var gotBinding *v1.Binding 57 | client := fake.NewSimpleClientset(testPod) 58 | client.PrependReactor("create", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { 59 | if action.GetSubresource() != "binding" { 60 | return false, nil, nil 61 | } 62 | if tt.injectErr != nil { 63 | return true, nil, tt.injectErr 64 | } 65 | gotBinding = action.(clienttesting.CreateAction).GetObject().(*v1.Binding) 66 | return true, gotBinding, nil 67 | }) 68 | 69 | fh, err := frameworkruntime.NewFramework(nil, nil, frameworkruntime.WithClientSet(client)) 70 | if err != nil { 71 | t.Fatal(err) 72 | } 73 | binder := &DefaultBinder{handle: fh} 74 | status := binder.Bind(context.Background(), nil, testPod, "foohost.kubernetes.mydomain.com") 75 | if got := status.AsError(); (tt.injectErr != nil) != (got != nil) { 76 | t.Errorf("got error %q, want %q", got, tt.injectErr) 77 | } 78 | if diff := cmp.Diff(tt.wantBinding, gotBinding); diff != "" { 79 | t.Errorf("got different binding (-want, +got): %s", diff) 80 | } 81 | }) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package noderesources 18 | 19 | import ( 20 | "math" 21 | 22 | "k8s.io/kubernetes/pkg/scheduler/apis/config" 23 | "k8s.io/kubernetes/pkg/scheduler/framework" 24 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" 25 | ) 26 | 27 | const ( 28 | maxUtilization = 100 29 | ) 30 | 31 | // buildRequestedToCapacityRatioScorerFunction allows users to apply bin packing 32 | // on core resources like CPU, Memory as well as extended resources like accelerators. 33 | func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resourceToWeightMap resourceToWeightMap) func(resourceToValueMap, resourceToValueMap) int64 { 34 | rawScoringFunction := helper.BuildBrokenLinearFunction(scoringFunctionShape) 35 | resourceScoringFunction := func(requested, capacity int64) int64 { 36 | if capacity == 0 || requested > capacity { 37 | return rawScoringFunction(maxUtilization) 38 | } 39 | 40 | return rawScoringFunction(requested * maxUtilization / capacity) 41 | } 42 | return func(requested, allocable resourceToValueMap) int64 { 43 | var nodeScore, weightSum int64 44 | for resource := range requested { 45 | weight := resourceToWeightMap[resource] 46 | resourceScore := resourceScoringFunction(requested[resource], allocable[resource]) 47 | if resourceScore > 0 { 48 | nodeScore += resourceScore * weight 49 | weightSum += weight 50 | } 51 | } 52 | if weightSum == 0 { 53 | return 0 54 | } 55 | return int64(math.Round(float64(nodeScore) / float64(weightSum))) 56 | } 57 | } 58 | 59 | func requestedToCapacityRatioScorer(weightMap resourceToWeightMap, shape []config.UtilizationShapePoint) func(resourceToValueMap, resourceToValueMap) int64 { 60 | shapes := make([]helper.FunctionShapePoint, 0, len(shape)) 61 | for _, point := range shape { 62 | shapes = append(shapes, helper.FunctionShapePoint{ 63 | Utilization: int64(point.Utilization), 64 | // MaxCustomPriorityScore may diverge from the max score used in the scheduler and defined by MaxNodeScore, 65 | // therefore we need to scale the score returned by requested to capacity ratio to the score range 66 | // used by the scheduler. 67 | Score: int64(point.Score) * (framework.MaxNodeScore / config.MaxCustomPriorityScore), 68 | }) 69 | } 70 | 71 | return buildRequestedToCapacityRatioScorerFunction(shapes, weightMap) 72 | } 73 | -------------------------------------------------------------------------------- /scheduler/internal/cache/fake/fake_cache.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package fake 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | "k8s.io/kubernetes/pkg/scheduler/framework" 22 | internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" 23 | ) 24 | 25 | // Cache is used for testing 26 | type Cache struct { 27 | AssumeFunc func(*v1.Pod) 28 | ForgetFunc func(*v1.Pod) 29 | IsAssumedPodFunc func(*v1.Pod) bool 30 | GetPodFunc func(*v1.Pod) *v1.Pod 31 | } 32 | 33 | // AssumePod is a fake method for testing. 34 | func (c *Cache) AssumePod(pod *v1.Pod) error { 35 | c.AssumeFunc(pod) 36 | return nil 37 | } 38 | 39 | // FinishBinding is a fake method for testing. 40 | func (c *Cache) FinishBinding(pod *v1.Pod) error { return nil } 41 | 42 | // ForgetPod is a fake method for testing. 43 | func (c *Cache) ForgetPod(pod *v1.Pod) error { 44 | c.ForgetFunc(pod) 45 | return nil 46 | } 47 | 48 | // AddPod is a fake method for testing. 49 | func (c *Cache) AddPod(pod *v1.Pod) error { return nil } 50 | 51 | // UpdatePod is a fake method for testing. 52 | func (c *Cache) UpdatePod(oldPod, newPod *v1.Pod) error { return nil } 53 | 54 | // RemovePod is a fake method for testing. 55 | func (c *Cache) RemovePod(pod *v1.Pod) error { return nil } 56 | 57 | // IsAssumedPod is a fake method for testing. 58 | func (c *Cache) IsAssumedPod(pod *v1.Pod) (bool, error) { 59 | return c.IsAssumedPodFunc(pod), nil 60 | } 61 | 62 | // GetPod is a fake method for testing. 63 | func (c *Cache) GetPod(pod *v1.Pod) (*v1.Pod, error) { 64 | return c.GetPodFunc(pod), nil 65 | } 66 | 67 | // AddNode is a fake method for testing. 68 | func (c *Cache) AddNode(node *v1.Node) *framework.NodeInfo { return nil } 69 | 70 | // UpdateNode is a fake method for testing. 71 | func (c *Cache) UpdateNode(oldNode, newNode *v1.Node) *framework.NodeInfo { return nil } 72 | 73 | // RemoveNode is a fake method for testing. 74 | func (c *Cache) RemoveNode(node *v1.Node) error { return nil } 75 | 76 | // UpdateSnapshot is a fake method for testing. 77 | func (c *Cache) UpdateSnapshot(snapshot *internalcache.Snapshot) error { 78 | return nil 79 | } 80 | 81 | // NodeCount is a fake method for testing. 82 | func (c *Cache) NodeCount() int { return 0 } 83 | 84 | // PodCount is a fake method for testing. 85 | func (c *Cache) PodCount() (int, error) { return 0, nil } 86 | 87 | // Dump is a fake method for testing. 88 | func (c *Cache) Dump() *internalcache.Dump { 89 | return &internalcache.Dump{} 90 | } 91 | -------------------------------------------------------------------------------- /drs-monitor/usage.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import time 3 | import threading 4 | 5 | states = [] 6 | last_in = 0.0 7 | last_out = 0.0 8 | last_time = 0.0 9 | 10 | class MonitorThread(threading.Thread): 11 | def __init__(self, id, func): 12 | threading.Thread.__init__(self) 13 | self.id = id 14 | self.func = func 15 | 16 | def run(self): 17 | print('[INFO] Start the {} thread...'.format(self.id)) 18 | self.func() 19 | print('[INFO] Exit the {} thread!'.format(self.id)) 20 | 21 | def Cpu(): 22 | command = "top -b -n 2 -d 0.1 | awk -F ',' '/%Cpu/{print $4}'" 23 | state, data = subprocess.getstatusoutput(command) 24 | if state != 0: 25 | print('[ERROR] Command {} failed!'.format(command)) 26 | else: 27 | cpu_free = list(map(float, data.replace(' ', '').replace("id", '').split('\n')[1:])) 28 | cpu_usage = float('%.2f' % (100 - sum(cpu_free) / len(cpu_free))) 29 | # print(cpu_usage) 30 | return cpu_usage 31 | 32 | def Memory(): 33 | command = "head -n 3 /proc/meminfo" 34 | state, data = subprocess.getstatusoutput(command) 35 | if state != 0: 36 | print('[ERROR] Command {} failed!'.format(command)) 37 | else: 38 | data = data.replace(' ', '').split('\n') 39 | mem_total = int(data[0].split(':')[1][:-2]) 40 | mem_available = int(data[2].split(':')[1][:-2]) 41 | mem_usage = float('%.2f' % ((mem_total - mem_available) * 100 / mem_total)) 42 | # print(mem_usage) 43 | return mem_usage 44 | 45 | def Network(): 46 | now = time.time() 47 | command = "awk '/ens33/{print $2,$10}' /proc/net/dev" 48 | state, net = subprocess.getstatusoutput(command) 49 | now_in, now_out = [int(item) for item in net.split(' ')] 50 | return now_in, now_out, now 51 | 52 | def Io(): 53 | command = "sudo iotop -k -P -o -n 2 -b | awk '/Total/{print $4,$5,$10,$11}'" 54 | state, data = subprocess.getstatusoutput(command) 55 | if state != 0: 56 | print('[ERROR] Command {} failed!'.format(command)) 57 | else: 58 | data = data.split('\n')[1].split(' ') 59 | if data[1] == 'M/s': 60 | read = float(data[0]) * 1024 61 | else: 62 | read = float(data[0]) 63 | if data[3] == 'M/s': 64 | write = float(data[2]) * 1024 65 | else: 66 | write = float(data[2]) 67 | # print(read, write) 68 | return read, write 69 | 70 | def state(): 71 | cpu = Cpu() 72 | mem = Memory() 73 | now_in, now_out, now = Network() 74 | read, write = Io() 75 | recv = float('%.2f' % ((now_in - last_in) / (now - last_time) / 1024)) 76 | tran = float('%.2f' % ((now_out - last_out) / (now - last_time) / 1024)) 77 | states.append([cpu, mem, recv, tran, read, write]) 78 | 79 | if __name__ == "__main__": 80 | 81 | last_in, last_out, last_time = Network() 82 | 83 | for i in range(120): 84 | thread = MonitorThread(i, state) 85 | thread.start() 86 | time.sleep(0.5) 87 | 88 | time.sleep(10) 89 | 90 | with open('usage.txt', 'w') as f: 91 | for s in states: 92 | f.write(str(s)[1:-1].replace(',', '') + '\n') 93 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/helper/spread_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helper 18 | 19 | import ( 20 | "fmt" 21 | "testing" 22 | "time" 23 | 24 | "github.com/google/go-cmp/cmp" 25 | v1 "k8s.io/api/core/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/client-go/informers" 28 | "k8s.io/client-go/kubernetes/fake" 29 | ) 30 | 31 | func TestGetPodServices(t *testing.T) { 32 | fakeInformerFactory := informers.NewSharedInformerFactory(&fake.Clientset{}, 0*time.Second) 33 | var services []*v1.Service 34 | for i := 0; i < 3; i++ { 35 | service := &v1.Service{ 36 | ObjectMeta: metav1.ObjectMeta{ 37 | Name: fmt.Sprintf("service-%d", i), 38 | Namespace: "test", 39 | }, 40 | Spec: v1.ServiceSpec{ 41 | Selector: map[string]string{ 42 | "app": fmt.Sprintf("test-%d", i), 43 | }, 44 | }, 45 | } 46 | services = append(services, service) 47 | fakeInformerFactory.Core().V1().Services().Informer().GetStore().Add(service) 48 | } 49 | var pods []*v1.Pod 50 | for i := 0; i < 5; i++ { 51 | pod := &v1.Pod{ 52 | ObjectMeta: metav1.ObjectMeta{ 53 | Namespace: "test", 54 | Name: fmt.Sprintf("test-pod-%d", i), 55 | Labels: map[string]string{ 56 | "app": fmt.Sprintf("test-%d", i), 57 | "label": fmt.Sprintf("label-%d", i), 58 | }, 59 | }, 60 | } 61 | pods = append(pods, pod) 62 | } 63 | 64 | tests := []struct { 65 | name string 66 | pod *v1.Pod 67 | expect []*v1.Service 68 | }{ 69 | { 70 | name: "GetPodServices for pod-0", 71 | pod: pods[0], 72 | expect: []*v1.Service{services[0]}, 73 | }, 74 | { 75 | name: "GetPodServices for pod-1", 76 | pod: pods[1], 77 | expect: []*v1.Service{services[1]}, 78 | }, 79 | { 80 | name: "GetPodServices for pod-2", 81 | pod: pods[2], 82 | expect: []*v1.Service{services[2]}, 83 | }, 84 | { 85 | name: "GetPodServices for pod-3", 86 | pod: pods[3], 87 | expect: nil, 88 | }, 89 | { 90 | name: "GetPodServices for pod-4", 91 | pod: pods[4], 92 | expect: nil, 93 | }, 94 | } 95 | for _, test := range tests { 96 | t.Run(test.name, func(t *testing.T) { 97 | get, err := GetPodServices(fakeInformerFactory.Core().V1().Services().Lister(), test.pod) 98 | if err != nil { 99 | t.Errorf("Error from GetPodServices: %v", err) 100 | } else if diff := cmp.Diff(test.expect, get); diff != "" { 101 | t.Errorf("Unexpected services (-want, +got):\n%s", diff) 102 | } 103 | }) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /scheduler/internal/cache/debugger/dumper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package debugger 18 | 19 | import ( 20 | "fmt" 21 | "strings" 22 | 23 | "k8s.io/klog/v2" 24 | 25 | "k8s.io/api/core/v1" 26 | "k8s.io/kubernetes/pkg/scheduler/framework" 27 | internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" 28 | "k8s.io/kubernetes/pkg/scheduler/internal/queue" 29 | ) 30 | 31 | // CacheDumper writes some information from the scheduler cache and the scheduling queue to the 32 | // scheduler logs for debugging purposes. 33 | type CacheDumper struct { 34 | cache internalcache.Cache 35 | podQueue queue.SchedulingQueue 36 | } 37 | 38 | // DumpAll writes cached nodes and scheduling queue information to the scheduler logs. 39 | func (d *CacheDumper) DumpAll() { 40 | d.dumpNodes() 41 | d.dumpSchedulingQueue() 42 | } 43 | 44 | // dumpNodes writes NodeInfo to the scheduler logs. 45 | func (d *CacheDumper) dumpNodes() { 46 | dump := d.cache.Dump() 47 | klog.InfoS("Dump of cached NodeInfo") 48 | for name, nodeInfo := range dump.Nodes { 49 | klog.Info(d.printNodeInfo(name, nodeInfo)) 50 | } 51 | } 52 | 53 | // dumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs. 54 | func (d *CacheDumper) dumpSchedulingQueue() { 55 | pendingPods := d.podQueue.PendingPods() 56 | var podData strings.Builder 57 | for _, p := range pendingPods { 58 | podData.WriteString(printPod(p)) 59 | } 60 | klog.Infof("Dump of scheduling queue:\n%s", podData.String()) 61 | } 62 | 63 | // printNodeInfo writes parts of NodeInfo to a string. 64 | func (d *CacheDumper) printNodeInfo(name string, n *framework.NodeInfo) string { 65 | var nodeData strings.Builder 66 | nodeData.WriteString(fmt.Sprintf("\nNode name: %s\nDeleted: %t\nRequested Resources: %+v\nAllocatable Resources:%+v\nScheduled Pods(number: %v):\n", 67 | name, n.Node() == nil, n.Requested, n.Allocatable, len(n.Pods))) 68 | // Dumping Pod Info 69 | for _, p := range n.Pods { 70 | nodeData.WriteString(printPod(p.Pod)) 71 | } 72 | // Dumping nominated pods info on the node 73 | nominatedPodInfos := d.podQueue.NominatedPodsForNode(name) 74 | if len(nominatedPodInfos) != 0 { 75 | nodeData.WriteString(fmt.Sprintf("Nominated Pods(number: %v):\n", len(nominatedPodInfos))) 76 | for _, pi := range nominatedPodInfos { 77 | nodeData.WriteString(printPod(pi.Pod)) 78 | } 79 | } 80 | return nodeData.String() 81 | } 82 | 83 | // printPod writes parts of a Pod object to a string. 84 | func printPod(p *v1.Pod) string { 85 | return fmt.Sprintf("name: %v, namespace: %v, uid: %v, phase: %v, nominated node: %v\n", p.Name, p.Namespace, p.UID, p.Status.Phase, p.Status.NominatedNodeName) 86 | } 87 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/nodeunschedulable/node_unschedulable.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nodeunschedulable 18 | 19 | import ( 20 | "context" 21 | 22 | v1 "k8s.io/api/core/v1" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | v1helper "k8s.io/component-helpers/scheduling/corev1" 25 | "k8s.io/kubernetes/pkg/scheduler/framework" 26 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" 27 | ) 28 | 29 | // NodeUnschedulable plugin filters nodes that set node.Spec.Unschedulable=true unless 30 | // the pod tolerates {key=node.kubernetes.io/unschedulable, effect:NoSchedule} taint. 31 | type NodeUnschedulable struct { 32 | } 33 | 34 | var _ framework.FilterPlugin = &NodeUnschedulable{} 35 | var _ framework.EnqueueExtensions = &NodeUnschedulable{} 36 | 37 | // Name is the name of the plugin used in the plugin registry and configurations. 38 | const Name = names.NodeUnschedulable 39 | 40 | const ( 41 | // ErrReasonUnknownCondition is used for NodeUnknownCondition predicate error. 42 | ErrReasonUnknownCondition = "node(s) had unknown conditions" 43 | // ErrReasonUnschedulable is used for NodeUnschedulable predicate error. 44 | ErrReasonUnschedulable = "node(s) were unschedulable" 45 | ) 46 | 47 | // EventsToRegister returns the possible events that may make a Pod 48 | // failed by this plugin schedulable. 49 | func (pl *NodeUnschedulable) EventsToRegister() []framework.ClusterEvent { 50 | return []framework.ClusterEvent{ 51 | {Resource: framework.Node, ActionType: framework.Add | framework.UpdateNodeTaint}, 52 | } 53 | } 54 | 55 | // Name returns name of the plugin. It is used in logs, etc. 56 | func (pl *NodeUnschedulable) Name() string { 57 | return Name 58 | } 59 | 60 | // Filter invoked at the filter extension point. 61 | func (pl *NodeUnschedulable) Filter(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 62 | if nodeInfo == nil || nodeInfo.Node() == nil { 63 | return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonUnknownCondition) 64 | } 65 | // If pod tolerate unschedulable taint, it's also tolerate `node.Spec.Unschedulable`. 66 | podToleratesUnschedulable := v1helper.TolerationsTolerateTaint(pod.Spec.Tolerations, &v1.Taint{ 67 | Key: v1.TaintNodeUnschedulable, 68 | Effect: v1.TaintEffectNoSchedule, 69 | }) 70 | // TODO (k82cn): deprecates `node.Spec.Unschedulable` in 1.13. 71 | if nodeInfo.Node().Spec.Unschedulable && !podToleratesUnschedulable { 72 | return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonUnschedulable) 73 | } 74 | return nil 75 | } 76 | 77 | // New initializes a new plugin and returns it. 78 | func New(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) { 79 | return &NodeUnschedulable{}, nil 80 | } 81 | -------------------------------------------------------------------------------- /scheduler/framework/extender.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package framework 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | extenderv1 "k8s.io/kube-scheduler/extender/v1" 22 | ) 23 | 24 | // Extender is an interface for external processes to influence scheduling 25 | // decisions made by Kubernetes. This is typically needed for resources not directly 26 | // managed by Kubernetes. 27 | type Extender interface { 28 | // Name returns a unique name that identifies the extender. 29 | Name() string 30 | 31 | // Filter based on extender-implemented predicate functions. The filtered list is 32 | // expected to be a subset of the supplied list. 33 | // The failedNodes and failedAndUnresolvableNodes optionally contains the list 34 | // of failed nodes and failure reasons, except nodes in the latter are 35 | // unresolvable. 36 | Filter(pod *v1.Pod, nodes []*v1.Node) (filteredNodes []*v1.Node, failedNodesMap extenderv1.FailedNodesMap, failedAndUnresolvable extenderv1.FailedNodesMap, err error) 37 | 38 | // Prioritize based on extender-implemented priority functions. The returned scores & weight 39 | // are used to compute the weighted score for an extender. The weighted scores are added to 40 | // the scores computed by Kubernetes scheduler. The total scores are used to do the host selection. 41 | Prioritize(pod *v1.Pod, nodes []*v1.Node) (hostPriorities *extenderv1.HostPriorityList, weight int64, err error) 42 | 43 | // Bind delegates the action of binding a pod to a node to the extender. 44 | Bind(binding *v1.Binding) error 45 | 46 | // IsBinder returns whether this extender is configured for the Bind method. 47 | IsBinder() bool 48 | 49 | // IsInterested returns true if at least one extended resource requested by 50 | // this pod is managed by this extender. 51 | IsInterested(pod *v1.Pod) bool 52 | 53 | // ProcessPreemption returns nodes with their victim pods processed by extender based on 54 | // given: 55 | // 1. Pod to schedule 56 | // 2. Candidate nodes and victim pods (nodeNameToVictims) generated by previous scheduling process. 57 | // The possible changes made by extender may include: 58 | // 1. Subset of given candidate nodes after preemption phase of extender. 59 | // 2. A different set of victim pod for every given candidate node after preemption phase of extender. 60 | ProcessPreemption( 61 | pod *v1.Pod, 62 | nodeNameToVictims map[string]*extenderv1.Victims, 63 | nodeInfos NodeInfoLister, 64 | ) (map[string]*extenderv1.Victims, error) 65 | 66 | // SupportsPreemption returns if the scheduler extender support preemption or not. 67 | SupportsPreemption() bool 68 | 69 | // IsIgnorable returns true indicates scheduling should not fail when this extender 70 | // is unavailable. This gives scheduler ability to fail fast and tolerate non-critical extenders as well. 71 | IsIgnorable() bool 72 | } 73 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/examples/multipoint/multipoint.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package multipoint 18 | 19 | import ( 20 | "context" 21 | 22 | v1 "k8s.io/api/core/v1" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | "k8s.io/kubernetes/pkg/scheduler/framework" 25 | ) 26 | 27 | // CommunicatingPlugin is an example of a plugin that implements two 28 | // extension points. It communicates through state with another function. 29 | type CommunicatingPlugin struct{} 30 | 31 | var _ framework.ReservePlugin = CommunicatingPlugin{} 32 | var _ framework.PreBindPlugin = CommunicatingPlugin{} 33 | 34 | // Name is the name of the plugin used in Registry and configurations. 35 | const Name = "multipoint-communicating-plugin" 36 | 37 | // Name returns name of the plugin. It is used in logs, etc. 38 | func (mc CommunicatingPlugin) Name() string { 39 | return Name 40 | } 41 | 42 | type stateData struct { 43 | data string 44 | } 45 | 46 | func (s *stateData) Clone() framework.StateData { 47 | copy := &stateData{ 48 | data: s.data, 49 | } 50 | return copy 51 | } 52 | 53 | // Reserve is the function invoked by the framework at "reserve" extension point. 54 | func (mc CommunicatingPlugin) Reserve(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) *framework.Status { 55 | if pod == nil { 56 | return framework.NewStatus(framework.Error, "pod cannot be nil") 57 | } 58 | if pod.Name == "my-test-pod" { 59 | state.Write(framework.StateKey(pod.Name), &stateData{data: "never bind"}) 60 | } 61 | return nil 62 | } 63 | 64 | // Unreserve is the function invoked by the framework when any error happens 65 | // during "reserve" extension point or later. 66 | func (mc CommunicatingPlugin) Unreserve(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) { 67 | if pod.Name == "my-test-pod" { 68 | // The pod is at the end of its lifecycle -- let's clean up the allocated 69 | // resources. In this case, our clean up is simply deleting the key written 70 | // in the Reserve operation. 71 | state.Delete(framework.StateKey(pod.Name)) 72 | } 73 | } 74 | 75 | // PreBind is the function invoked by the framework at "prebind" extension point. 76 | func (mc CommunicatingPlugin) PreBind(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) *framework.Status { 77 | if pod == nil { 78 | return framework.NewStatus(framework.Error, "pod cannot be nil") 79 | } 80 | if v, e := state.Read(framework.StateKey(pod.Name)); e == nil { 81 | if value, ok := v.(*stateData); ok && value.data == "never bind" { 82 | return framework.NewStatus(framework.Unschedulable, "pod is not permitted") 83 | } 84 | } 85 | return nil 86 | } 87 | 88 | // New initializes a new plugin and returns it. 89 | func New(_ *runtime.Unknown, _ framework.Handle) (framework.Plugin, error) { 90 | return &CommunicatingPlugin{}, nil 91 | } 92 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta2/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Code generated by defaulter-gen. DO NOT EDIT. 21 | 22 | package v1beta2 23 | 24 | import ( 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | v1beta2 "k8s.io/kube-scheduler/config/v1beta2" 27 | ) 28 | 29 | // RegisterDefaults adds defaulters functions to the given scheme. 30 | // Public to allow building arbitrary schemes. 31 | // All generated defaulters are covering - they call all nested defaulters. 32 | func RegisterDefaults(scheme *runtime.Scheme) error { 33 | scheme.AddTypeDefaultingFunc(&v1beta2.DefaultPreemptionArgs{}, func(obj interface{}) { SetObjectDefaults_DefaultPreemptionArgs(obj.(*v1beta2.DefaultPreemptionArgs)) }) 34 | scheme.AddTypeDefaultingFunc(&v1beta2.InterPodAffinityArgs{}, func(obj interface{}) { SetObjectDefaults_InterPodAffinityArgs(obj.(*v1beta2.InterPodAffinityArgs)) }) 35 | scheme.AddTypeDefaultingFunc(&v1beta2.KubeSchedulerConfiguration{}, func(obj interface{}) { 36 | SetObjectDefaults_KubeSchedulerConfiguration(obj.(*v1beta2.KubeSchedulerConfiguration)) 37 | }) 38 | scheme.AddTypeDefaultingFunc(&v1beta2.NodeResourcesBalancedAllocationArgs{}, func(obj interface{}) { 39 | SetObjectDefaults_NodeResourcesBalancedAllocationArgs(obj.(*v1beta2.NodeResourcesBalancedAllocationArgs)) 40 | }) 41 | scheme.AddTypeDefaultingFunc(&v1beta2.NodeResourcesFitArgs{}, func(obj interface{}) { SetObjectDefaults_NodeResourcesFitArgs(obj.(*v1beta2.NodeResourcesFitArgs)) }) 42 | scheme.AddTypeDefaultingFunc(&v1beta2.PodTopologySpreadArgs{}, func(obj interface{}) { SetObjectDefaults_PodTopologySpreadArgs(obj.(*v1beta2.PodTopologySpreadArgs)) }) 43 | scheme.AddTypeDefaultingFunc(&v1beta2.VolumeBindingArgs{}, func(obj interface{}) { SetObjectDefaults_VolumeBindingArgs(obj.(*v1beta2.VolumeBindingArgs)) }) 44 | return nil 45 | } 46 | 47 | func SetObjectDefaults_DefaultPreemptionArgs(in *v1beta2.DefaultPreemptionArgs) { 48 | SetDefaults_DefaultPreemptionArgs(in) 49 | } 50 | 51 | func SetObjectDefaults_InterPodAffinityArgs(in *v1beta2.InterPodAffinityArgs) { 52 | SetDefaults_InterPodAffinityArgs(in) 53 | } 54 | 55 | func SetObjectDefaults_KubeSchedulerConfiguration(in *v1beta2.KubeSchedulerConfiguration) { 56 | SetDefaults_KubeSchedulerConfiguration(in) 57 | } 58 | 59 | func SetObjectDefaults_NodeResourcesBalancedAllocationArgs(in *v1beta2.NodeResourcesBalancedAllocationArgs) { 60 | SetDefaults_NodeResourcesBalancedAllocationArgs(in) 61 | } 62 | 63 | func SetObjectDefaults_NodeResourcesFitArgs(in *v1beta2.NodeResourcesFitArgs) { 64 | SetDefaults_NodeResourcesFitArgs(in) 65 | } 66 | 67 | func SetObjectDefaults_PodTopologySpreadArgs(in *v1beta2.PodTopologySpreadArgs) { 68 | SetDefaults_PodTopologySpreadArgs(in) 69 | } 70 | 71 | func SetObjectDefaults_VolumeBindingArgs(in *v1beta2.VolumeBindingArgs) { 72 | SetDefaults_VolumeBindingArgs(in) 73 | } 74 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta3/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Code generated by defaulter-gen. DO NOT EDIT. 21 | 22 | package v1beta3 23 | 24 | import ( 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | v1beta3 "k8s.io/kube-scheduler/config/v1beta3" 27 | ) 28 | 29 | // RegisterDefaults adds defaulters functions to the given scheme. 30 | // Public to allow building arbitrary schemes. 31 | // All generated defaulters are covering - they call all nested defaulters. 32 | func RegisterDefaults(scheme *runtime.Scheme) error { 33 | scheme.AddTypeDefaultingFunc(&v1beta3.DefaultPreemptionArgs{}, func(obj interface{}) { SetObjectDefaults_DefaultPreemptionArgs(obj.(*v1beta3.DefaultPreemptionArgs)) }) 34 | scheme.AddTypeDefaultingFunc(&v1beta3.InterPodAffinityArgs{}, func(obj interface{}) { SetObjectDefaults_InterPodAffinityArgs(obj.(*v1beta3.InterPodAffinityArgs)) }) 35 | scheme.AddTypeDefaultingFunc(&v1beta3.KubeSchedulerConfiguration{}, func(obj interface{}) { 36 | SetObjectDefaults_KubeSchedulerConfiguration(obj.(*v1beta3.KubeSchedulerConfiguration)) 37 | }) 38 | scheme.AddTypeDefaultingFunc(&v1beta3.NodeResourcesBalancedAllocationArgs{}, func(obj interface{}) { 39 | SetObjectDefaults_NodeResourcesBalancedAllocationArgs(obj.(*v1beta3.NodeResourcesBalancedAllocationArgs)) 40 | }) 41 | scheme.AddTypeDefaultingFunc(&v1beta3.NodeResourcesFitArgs{}, func(obj interface{}) { SetObjectDefaults_NodeResourcesFitArgs(obj.(*v1beta3.NodeResourcesFitArgs)) }) 42 | scheme.AddTypeDefaultingFunc(&v1beta3.PodTopologySpreadArgs{}, func(obj interface{}) { SetObjectDefaults_PodTopologySpreadArgs(obj.(*v1beta3.PodTopologySpreadArgs)) }) 43 | scheme.AddTypeDefaultingFunc(&v1beta3.VolumeBindingArgs{}, func(obj interface{}) { SetObjectDefaults_VolumeBindingArgs(obj.(*v1beta3.VolumeBindingArgs)) }) 44 | return nil 45 | } 46 | 47 | func SetObjectDefaults_DefaultPreemptionArgs(in *v1beta3.DefaultPreemptionArgs) { 48 | SetDefaults_DefaultPreemptionArgs(in) 49 | } 50 | 51 | func SetObjectDefaults_InterPodAffinityArgs(in *v1beta3.InterPodAffinityArgs) { 52 | SetDefaults_InterPodAffinityArgs(in) 53 | } 54 | 55 | func SetObjectDefaults_KubeSchedulerConfiguration(in *v1beta3.KubeSchedulerConfiguration) { 56 | SetDefaults_KubeSchedulerConfiguration(in) 57 | } 58 | 59 | func SetObjectDefaults_NodeResourcesBalancedAllocationArgs(in *v1beta3.NodeResourcesBalancedAllocationArgs) { 60 | SetDefaults_NodeResourcesBalancedAllocationArgs(in) 61 | } 62 | 63 | func SetObjectDefaults_NodeResourcesFitArgs(in *v1beta3.NodeResourcesFitArgs) { 64 | SetDefaults_NodeResourcesFitArgs(in) 65 | } 66 | 67 | func SetObjectDefaults_PodTopologySpreadArgs(in *v1beta3.PodTopologySpreadArgs) { 68 | SetDefaults_PodTopologySpreadArgs(in) 69 | } 70 | 71 | func SetObjectDefaults_VolumeBindingArgs(in *v1beta3.VolumeBindingArgs) { 72 | SetDefaults_VolumeBindingArgs(in) 73 | } 74 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DRS 2 | A Deep Reinforcement Learning enhanced Kubernetes Scheduler for Microservice-based System 3 | 4 | ## File description 5 | - `scheduler/` : The source code of kube-scheduler for [Kubernetes (version 1.23.4)](https://github.com/kubernetes/kubernetes/tree/v1.23.4). The DRS scheduler is in registed in `scheduler/framework/plugins/dqn/dqn.go` 6 | - `deploy/` : 7 | - `apps/` : the application configure file and deploy script. 8 | 9 | The docker images of the applications in available on DockerHub. 10 | 11 | | Application | Type | Description | Docker Image | 12 | | :----: | :----: | :----: | :----: | 13 | | Video Scale | CPU-intensive | Scale the video to a certain size with ffmpeg | [jolyonjian/apps:cpu-1.0](https://hub.docker.com/repository/docker/jolyonjian/apps) | 14 | | Transmission | Network-intensive | Transfer data of a certain size to the server | [jolyonjian/apps:net-1.0](https://hub.docker.com/repository/docker/jolyonjian/apps) | 15 | | Data Write | IO-intensive | Read a file on the disk and write a copy | [jolyonjian/apps:io-1.0](https://hub.docker.com/repository/docker/jolyonjian/apps) | 16 | 17 | - `scripts/` : Scripts for cluster creation, initialization, deletion, etc. 18 | - `drs-scheduler/` : DRS scheduler runs on the master node of the k8s cluster 19 | - `drs-monitor/` : DRS monitor runs on the worker node of the k8s cluster 20 | 21 | ## Run 22 | 1. Modify the source code of [Kubernetes (version 1.23.4)](https://github.com/kubernetes/kubernetes/tree/v1.23.4) to regist DRS scheduler and compile the project. 23 | ``` 24 | # Configure go environment 25 | 26 | # clone the source code of Kubernetes v1.23.4 27 | $ git clone -b v1.23.4 https://github.com/kubernetes/kubernetes.git 28 | $ mv ./kubernetes / 29 | 30 | $ git clone https://github.com/JolyonJian/DRS 31 | $ cd DRS 32 | # Replace the source code of Kube-scheduler 33 | $ mv ./scheduler /kubernetes/pkg/ 34 | $ cd /kubernetes 35 | $ make 36 | 37 | # After the first compilation you can only compile the kube-scheduler 38 | $ make cmd/kube-scheduler 39 | ``` 40 | 2. Initalize the Kubernetes cluster. 41 | ``` 42 | # Start a k8s cluster (on the master node) 43 | $ cd /deploy/scripts 44 | $ ./init.sh 45 | $ ./env.sh 46 | 47 | # Add the worker nodes into the cluster (on each worker node) 48 | $ kubeadm join :6443 --token --discovery-token-ca-cert-hash 49 | 50 | # Deploy the network and the second scheduler plugins 51 | $ cd /deploy/apps 52 | $ ./apply.sh kube-flannel.yaml 53 | $ ./apply.sh drs-scheduler.yaml 54 | ``` 55 | 3. Start the DRS scheduler and DRS the monitor. 56 | ``` 57 | # Start the DRS scheduler (on the master node) 58 | $ cd /scheduler 59 | # The node ip needs to be configured according to your environment 60 | $ python dqn.py 61 | 62 | # Start the DRS monitor (on each worker node) 63 | $ cd /monitor 64 | # The node ip and port need to be configured according to your environment 65 | $ ./monitor.sh 66 | ``` 67 | 68 | 4. Deploy applications to the cluster. 69 | ``` 70 | $ cd /deploy/apps 71 | # Specify the scheduler in the configuration file 72 | $ ./apply.sh 73 | ``` 74 | 75 | ## Contact 76 | The link of our paper (Under Review): [https://www.authorea.com/doi/full/10.22541/au.167285897.72278925](https://www.authorea.com/doi/full/10.22541/au.167285897.72278925) 77 | 78 | If you have any questions, please contact us. 79 | 80 | Zhaolong Jian: jianzhaolong@mail.nankai.edu.cn 81 | -------------------------------------------------------------------------------- /scheduler/util/pod_resources.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | utilfeature "k8s.io/apiserver/pkg/util/feature" 22 | "k8s.io/kubernetes/pkg/features" 23 | ) 24 | 25 | // For each of these resources, a pod that doesn't request the resource explicitly 26 | // will be treated as having requested the amount indicated below, for the purpose 27 | // of computing priority only. This ensures that when scheduling zero-request pods, such 28 | // pods will not all be scheduled to the machine with the smallest in-use request, 29 | // and that when scheduling regular pods, such pods will not see zero-request pods as 30 | // consuming no resources whatsoever. We chose these values to be similar to the 31 | // resources that we give to cluster addon pods (#10653). But they are pretty arbitrary. 32 | // As described in #11713, we use request instead of limit to deal with resource requirements. 33 | const ( 34 | // DefaultMilliCPURequest defines default milli cpu request number. 35 | DefaultMilliCPURequest int64 = 100 // 0.1 core 36 | // DefaultMemoryRequest defines default memory request size. 37 | DefaultMemoryRequest int64 = 200 * 1024 * 1024 // 200 MB 38 | ) 39 | 40 | // GetNonzeroRequests returns the default cpu and memory resource request if none is found or 41 | // what is provided on the request. 42 | func GetNonzeroRequests(requests *v1.ResourceList) (int64, int64) { 43 | return GetRequestForResource(v1.ResourceCPU, requests, true), 44 | GetRequestForResource(v1.ResourceMemory, requests, true) 45 | } 46 | 47 | // GetRequestForResource returns the requested values unless nonZero is true and there is no defined request 48 | // for CPU and memory. 49 | // If nonZero is true and the resource has no defined request for CPU or memory, it returns a default value. 50 | func GetRequestForResource(resource v1.ResourceName, requests *v1.ResourceList, nonZero bool) int64 { 51 | if requests == nil { 52 | return 0 53 | } 54 | switch resource { 55 | case v1.ResourceCPU: 56 | // Override if un-set, but not if explicitly set to zero 57 | if _, found := (*requests)[v1.ResourceCPU]; !found && nonZero { 58 | return DefaultMilliCPURequest 59 | } 60 | return requests.Cpu().MilliValue() 61 | case v1.ResourceMemory: 62 | // Override if un-set, but not if explicitly set to zero 63 | if _, found := (*requests)[v1.ResourceMemory]; !found && nonZero { 64 | return DefaultMemoryRequest 65 | } 66 | return requests.Memory().Value() 67 | case v1.ResourceEphemeralStorage: 68 | // if the local storage capacity isolation feature gate is disabled, pods request 0 disk. 69 | if !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) { 70 | return 0 71 | } 72 | 73 | quantity, found := (*requests)[v1.ResourceEphemeralStorage] 74 | if !found { 75 | return 0 76 | } 77 | return quantity.Value() 78 | default: 79 | quantity, found := (*requests)[resource] 80 | if !found { 81 | return 0 82 | } 83 | return quantity.Value() 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/queuesort/priority_sort_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package queuesort 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | "k8s.io/kubernetes/pkg/scheduler/framework" 22 | "testing" 23 | "time" 24 | ) 25 | 26 | func TestLess(t *testing.T) { 27 | prioritySort := &PrioritySort{} 28 | var lowPriority, highPriority = int32(10), int32(100) 29 | t1 := time.Now() 30 | t2 := t1.Add(time.Second) 31 | for _, tt := range []struct { 32 | name string 33 | p1 *framework.QueuedPodInfo 34 | p2 *framework.QueuedPodInfo 35 | expected bool 36 | }{ 37 | { 38 | name: "p1.priority less than p2.priority", 39 | p1: &framework.QueuedPodInfo{ 40 | PodInfo: framework.NewPodInfo(&v1.Pod{ 41 | Spec: v1.PodSpec{ 42 | Priority: &lowPriority, 43 | }, 44 | }), 45 | }, 46 | p2: &framework.QueuedPodInfo{ 47 | PodInfo: framework.NewPodInfo(&v1.Pod{ 48 | Spec: v1.PodSpec{ 49 | Priority: &highPriority, 50 | }, 51 | }), 52 | }, 53 | expected: false, // p2 should be ahead of p1 in the queue 54 | }, 55 | { 56 | name: "p1.priority greater than p2.priority", 57 | p1: &framework.QueuedPodInfo{ 58 | PodInfo: framework.NewPodInfo(&v1.Pod{ 59 | Spec: v1.PodSpec{ 60 | Priority: &highPriority, 61 | }, 62 | }), 63 | }, 64 | p2: &framework.QueuedPodInfo{ 65 | PodInfo: framework.NewPodInfo(&v1.Pod{ 66 | Spec: v1.PodSpec{ 67 | Priority: &lowPriority, 68 | }, 69 | }), 70 | }, 71 | expected: true, // p1 should be ahead of p2 in the queue 72 | }, 73 | { 74 | name: "equal priority. p1 is added to schedulingQ earlier than p2", 75 | p1: &framework.QueuedPodInfo{ 76 | PodInfo: framework.NewPodInfo(&v1.Pod{ 77 | Spec: v1.PodSpec{ 78 | Priority: &highPriority, 79 | }, 80 | }), 81 | Timestamp: t1, 82 | }, 83 | p2: &framework.QueuedPodInfo{ 84 | PodInfo: framework.NewPodInfo(&v1.Pod{ 85 | Spec: v1.PodSpec{ 86 | Priority: &highPriority, 87 | }, 88 | }), 89 | Timestamp: t2, 90 | }, 91 | expected: true, // p1 should be ahead of p2 in the queue 92 | }, 93 | { 94 | name: "equal priority. p2 is added to schedulingQ earlier than p1", 95 | p1: &framework.QueuedPodInfo{ 96 | PodInfo: framework.NewPodInfo(&v1.Pod{ 97 | Spec: v1.PodSpec{ 98 | Priority: &highPriority, 99 | }, 100 | }), 101 | Timestamp: t2, 102 | }, 103 | p2: &framework.QueuedPodInfo{ 104 | PodInfo: framework.NewPodInfo(&v1.Pod{ 105 | Spec: v1.PodSpec{ 106 | Priority: &highPriority, 107 | }, 108 | }), 109 | Timestamp: t1, 110 | }, 111 | expected: false, // p2 should be ahead of p1 in the queue 112 | }, 113 | } { 114 | t.Run(tt.name, func(t *testing.T) { 115 | if got := prioritySort.Less(tt.p1, tt.p2); got != tt.expected { 116 | t.Errorf("expected %v, got %v", tt.expected, got) 117 | } 118 | }) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/nodevolumelimits/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nodevolumelimits 18 | 19 | import ( 20 | "strings" 21 | 22 | "k8s.io/api/core/v1" 23 | storagev1 "k8s.io/api/storage/v1" 24 | "k8s.io/apimachinery/pkg/util/sets" 25 | utilfeature "k8s.io/apiserver/pkg/util/feature" 26 | csilibplugins "k8s.io/csi-translation-lib/plugins" 27 | v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" 28 | "k8s.io/kubernetes/pkg/features" 29 | "k8s.io/kubernetes/pkg/scheduler/framework" 30 | ) 31 | 32 | // isCSIMigrationOn returns a boolean value indicating whether 33 | // the CSI migration has been enabled for a particular storage plugin. 34 | func isCSIMigrationOn(csiNode *storagev1.CSINode, pluginName string) bool { 35 | if csiNode == nil || len(pluginName) == 0 { 36 | return false 37 | } 38 | 39 | // In-tree storage to CSI driver migration feature should be enabled, 40 | // along with the plugin-specific one 41 | if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { 42 | return false 43 | } 44 | 45 | switch pluginName { 46 | case csilibplugins.AWSEBSInTreePluginName: 47 | if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWS) { 48 | return false 49 | } 50 | case csilibplugins.PortworxVolumePluginName: 51 | if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx) { 52 | return false 53 | } 54 | case csilibplugins.GCEPDInTreePluginName: 55 | if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCE) { 56 | return false 57 | } 58 | case csilibplugins.AzureDiskInTreePluginName: 59 | if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureDisk) { 60 | return false 61 | } 62 | case csilibplugins.CinderInTreePluginName: 63 | if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStack) { 64 | return false 65 | } 66 | case csilibplugins.RBDVolumePluginName: 67 | if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationRBD) { 68 | return false 69 | } 70 | default: 71 | return false 72 | } 73 | 74 | // The plugin name should be listed in the CSINode object annotation. 75 | // This indicates that the plugin has been migrated to a CSI driver in the node. 76 | csiNodeAnn := csiNode.GetAnnotations() 77 | if csiNodeAnn == nil { 78 | return false 79 | } 80 | 81 | var mpaSet sets.String 82 | mpa := csiNodeAnn[v1.MigratedPluginsAnnotationKey] 83 | if len(mpa) == 0 { 84 | mpaSet = sets.NewString() 85 | } else { 86 | tok := strings.Split(mpa, ",") 87 | mpaSet = sets.NewString(tok...) 88 | } 89 | 90 | return mpaSet.Has(pluginName) 91 | } 92 | 93 | // volumeLimits returns volume limits associated with the node. 94 | func volumeLimits(n *framework.NodeInfo) map[v1.ResourceName]int64 { 95 | volumeLimits := map[v1.ResourceName]int64{} 96 | for k, v := range n.Allocatable.ScalarResources { 97 | if v1helper.IsAttachableVolumeResourceName(k) { 98 | volumeLimits[k] = v 99 | } 100 | } 101 | return volumeLimits 102 | } 103 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/selectorspread/selector_spread_perf_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package selectorspread 18 | 19 | import ( 20 | "context" 21 | "testing" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | "k8s.io/client-go/informers" 25 | "k8s.io/client-go/kubernetes/fake" 26 | "k8s.io/kubernetes/pkg/scheduler/framework" 27 | "k8s.io/kubernetes/pkg/scheduler/framework/parallelize" 28 | "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 29 | "k8s.io/kubernetes/pkg/scheduler/internal/cache" 30 | st "k8s.io/kubernetes/pkg/scheduler/testing" 31 | ) 32 | 33 | var ( 34 | tests = []struct { 35 | name string 36 | existingPodsNum int 37 | allNodesNum int 38 | }{ 39 | { 40 | name: "100nodes", 41 | existingPodsNum: 1000, 42 | allNodesNum: 100, 43 | }, 44 | { 45 | name: "1000nodes", 46 | existingPodsNum: 10000, 47 | allNodesNum: 1000, 48 | }, 49 | } 50 | ) 51 | 52 | func BenchmarkTestSelectorSpreadPriority(b *testing.B) { 53 | for _, tt := range tests { 54 | b.Run(tt.name, func(b *testing.B) { 55 | pod := st.MakePod().Name("p").Label("foo", "").Obj() 56 | existingPods, allNodes, filteredNodes := st.MakeNodesAndPodsForEvenPodsSpread(pod.Labels, tt.existingPodsNum, tt.allNodesNum, tt.allNodesNum) 57 | snapshot := cache.NewSnapshot(existingPods, allNodes) 58 | client := fake.NewSimpleClientset( 59 | &v1.Service{Spec: v1.ServiceSpec{Selector: map[string]string{"foo": ""}}}, 60 | ) 61 | ctx := context.Background() 62 | informerFactory := informers.NewSharedInformerFactory(client, 0) 63 | _ = informerFactory.Core().V1().Services().Lister() 64 | informerFactory.Start(ctx.Done()) 65 | caches := informerFactory.WaitForCacheSync(ctx.Done()) 66 | for _, synced := range caches { 67 | if !synced { 68 | b.Errorf("error waiting for informer cache sync") 69 | } 70 | } 71 | fh, _ := runtime.NewFramework(nil, nil, runtime.WithSnapshotSharedLister(snapshot), runtime.WithInformerFactory(informerFactory)) 72 | pl, err := New(nil, fh) 73 | if err != nil { 74 | b.Fatal(err) 75 | } 76 | plugin := pl.(*SelectorSpread) 77 | b.ResetTimer() 78 | 79 | for i := 0; i < b.N; i++ { 80 | state := framework.NewCycleState() 81 | status := plugin.PreScore(ctx, state, pod, allNodes) 82 | if !status.IsSuccess() { 83 | b.Fatalf("unexpected error: %v", status) 84 | } 85 | gotList := make(framework.NodeScoreList, len(filteredNodes)) 86 | scoreNode := func(i int) { 87 | n := filteredNodes[i] 88 | score, _ := plugin.Score(ctx, state, pod, n.Name) 89 | gotList[i] = framework.NodeScore{Name: n.Name, Score: score} 90 | } 91 | parallelizer := parallelize.NewParallelizer(parallelize.DefaultParallelism) 92 | parallelizer.Until(ctx, len(filteredNodes), scoreNode) 93 | status = plugin.NormalizeScore(ctx, state, pod, gotList) 94 | if !status.IsSuccess() { 95 | b.Fatal(status) 96 | } 97 | } 98 | }) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /deploy/apps/drs-scheduler.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | 3 | kind: ServiceAccount 4 | 5 | metadata: 6 | 7 | name: my-scheduler 8 | 9 | namespace: kube-system 10 | 11 | --- 12 | 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | 15 | kind: ClusterRoleBinding 16 | 17 | metadata: 18 | 19 | name: my-scheduler-as-kube-scheduler 20 | 21 | subjects: 22 | 23 | - kind: ServiceAccount 24 | 25 | name: my-scheduler 26 | 27 | namespace: kube-system 28 | 29 | roleRef: 30 | 31 | kind: ClusterRole 32 | 33 | name: system:kube-scheduler 34 | 35 | apiGroup: rbac.authorization.k8s.io 36 | 37 | --- 38 | 39 | apiVersion: v1 40 | 41 | kind: ConfigMap 42 | 43 | metadata: 44 | 45 | name: my-scheduler-config 46 | 47 | namespace: kube-system 48 | 49 | data: 50 | 51 | my-scheduler-config.yaml: | 52 | 53 | apiVersion: kubescheduler.config.k8s.io/v1beta2 54 | 55 | kind: KubeSchedulerConfiguration 56 | 57 | leaderElection: 58 | 59 | leaderElect: false 60 | 61 | profiles: 62 | 63 | - schedulerName: my-scheduler 64 | 65 | plugins: 66 | 67 | filter: 68 | 69 | enabled: 70 | 71 | - name: "dqn-plugin" 72 | --- 73 | 74 | apiVersion: rbac.authorization.k8s.io/v1 75 | 76 | kind: ClusterRoleBinding 77 | 78 | metadata: 79 | 80 | name: my-scheduler-as-volume-scheduler 81 | 82 | subjects: 83 | 84 | - kind: ServiceAccount 85 | 86 | name: my-scheduler 87 | 88 | namespace: kube-system 89 | 90 | roleRef: 91 | 92 | kind: ClusterRole 93 | 94 | name: system:volume-scheduler 95 | 96 | apiGroup: rbac.authorization.k8s.io 97 | 98 | --- 99 | 100 | apiVersion: apps/v1 101 | 102 | kind: Deployment 103 | 104 | metadata: 105 | 106 | labels: 107 | 108 | component: scheduler 109 | 110 | tier: control-plane 111 | 112 | name: my-scheduler 113 | 114 | namespace: kube-system 115 | 116 | spec: 117 | 118 | selector: 119 | 120 | matchLabels: 121 | 122 | component: scheduler 123 | 124 | tier: control-plane 125 | 126 | replicas: 1 127 | 128 | template: 129 | 130 | metadata: 131 | 132 | labels: 133 | 134 | component: scheduler 135 | 136 | tier: control-plane 137 | 138 | version: second 139 | 140 | spec: 141 | 142 | serviceAccountName: my-scheduler 143 | 144 | containers: 145 | 146 | - command: 147 | 148 | - /usr/local/bin/kube-scheduler 149 | 150 | - --config=/etc/kubernetes/my-scheduler/my-scheduler-config.yaml 151 | 152 | image: jolyonjian/my-scheduler:1.0 153 | 154 | imagePullPolicy: IfNotPresent 155 | 156 | livenessProbe: 157 | 158 | httpGet: 159 | 160 | path: /healthz 161 | 162 | port: 10259 163 | 164 | scheme: HTTPS 165 | 166 | initialDelaySeconds: 15 167 | 168 | name: kube-second-scheduler 169 | 170 | readinessProbe: 171 | 172 | httpGet: 173 | 174 | path: /healthz 175 | 176 | port: 10259 177 | 178 | scheme: HTTPS 179 | 180 | resources: 181 | 182 | requests: 183 | 184 | cpu: '0.1' 185 | 186 | securityContext: 187 | 188 | privileged: false 189 | 190 | volumeMounts: 191 | 192 | - name: config-volume 193 | 194 | mountPath: /etc/kubernetes/my-scheduler 195 | 196 | hostNetwork: false 197 | 198 | hostPID: false 199 | 200 | volumes: 201 | 202 | - name: config-volume 203 | 204 | configMap: 205 | 206 | name: my-scheduler-config 207 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/podtopologyspread/common.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package podtopologyspread 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/labels" 23 | "k8s.io/kubernetes/pkg/scheduler/framework" 24 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" 25 | ) 26 | 27 | type topologyPair struct { 28 | key string 29 | value string 30 | } 31 | 32 | // topologySpreadConstraint is an internal version for v1.TopologySpreadConstraint 33 | // and where the selector is parsed. 34 | // Fields are exported for comparison during testing. 35 | type topologySpreadConstraint struct { 36 | MaxSkew int32 37 | TopologyKey string 38 | Selector labels.Selector 39 | } 40 | 41 | // buildDefaultConstraints builds the constraints for a pod using 42 | // .DefaultConstraints and the selectors from the services, replication 43 | // controllers, replica sets and stateful sets that match the pod. 44 | func (pl *PodTopologySpread) buildDefaultConstraints(p *v1.Pod, action v1.UnsatisfiableConstraintAction) ([]topologySpreadConstraint, error) { 45 | constraints, err := filterTopologySpreadConstraints(pl.defaultConstraints, action) 46 | if err != nil || len(constraints) == 0 { 47 | return nil, err 48 | } 49 | selector := helper.DefaultSelector(p, pl.services, pl.replicationCtrls, pl.replicaSets, pl.statefulSets) 50 | if selector.Empty() { 51 | return nil, nil 52 | } 53 | for i := range constraints { 54 | constraints[i].Selector = selector 55 | } 56 | return constraints, nil 57 | } 58 | 59 | // nodeLabelsMatchSpreadConstraints checks if ALL topology keys in spread Constraints are present in node labels. 60 | func nodeLabelsMatchSpreadConstraints(nodeLabels map[string]string, constraints []topologySpreadConstraint) bool { 61 | for _, c := range constraints { 62 | if _, ok := nodeLabels[c.TopologyKey]; !ok { 63 | return false 64 | } 65 | } 66 | return true 67 | } 68 | 69 | func filterTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint, action v1.UnsatisfiableConstraintAction) ([]topologySpreadConstraint, error) { 70 | var result []topologySpreadConstraint 71 | for _, c := range constraints { 72 | if c.WhenUnsatisfiable == action { 73 | selector, err := metav1.LabelSelectorAsSelector(c.LabelSelector) 74 | if err != nil { 75 | return nil, err 76 | } 77 | result = append(result, topologySpreadConstraint{ 78 | MaxSkew: c.MaxSkew, 79 | TopologyKey: c.TopologyKey, 80 | Selector: selector, 81 | }) 82 | } 83 | } 84 | return result, nil 85 | } 86 | 87 | func countPodsMatchSelector(podInfos []*framework.PodInfo, selector labels.Selector, ns string) int { 88 | count := 0 89 | for _, p := range podInfos { 90 | // Bypass terminating Pod (see #87621). 91 | if p.Pod.DeletionTimestamp != nil || p.Pod.Namespace != ns { 92 | continue 93 | } 94 | if selector.Matches(labels.Set(p.Pod.Labels)) { 95 | count++ 96 | } 97 | } 98 | return count 99 | } 100 | -------------------------------------------------------------------------------- /scheduler/framework/runtime/metrics_recorder.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package runtime 18 | 19 | import ( 20 | "time" 21 | 22 | k8smetrics "k8s.io/component-base/metrics" 23 | "k8s.io/kubernetes/pkg/scheduler/framework" 24 | "k8s.io/kubernetes/pkg/scheduler/metrics" 25 | ) 26 | 27 | // frameworkMetric is the data structure passed in the buffer channel between the main framework thread 28 | // and the metricsRecorder goroutine. 29 | type frameworkMetric struct { 30 | metric *k8smetrics.HistogramVec 31 | labelValues []string 32 | value float64 33 | } 34 | 35 | // metricRecorder records framework metrics in a separate goroutine to avoid overhead in the critical path. 36 | type metricsRecorder struct { 37 | // bufferCh is a channel that serves as a metrics buffer before the metricsRecorder goroutine reports it. 38 | bufferCh chan *frameworkMetric 39 | // if bufferSize is reached, incoming metrics will be discarded. 40 | bufferSize int 41 | // how often the recorder runs to flush the metrics. 42 | interval time.Duration 43 | 44 | // stopCh is used to stop the goroutine which periodically flushes metrics. It's currently only 45 | // used in tests. 46 | stopCh chan struct{} 47 | // isStoppedCh indicates whether the goroutine is stopped. It's used in tests only to make sure 48 | // the metric flushing goroutine is stopped so that tests can collect metrics for verification. 49 | isStoppedCh chan struct{} 50 | } 51 | 52 | func newMetricsRecorder(bufferSize int, interval time.Duration) *metricsRecorder { 53 | recorder := &metricsRecorder{ 54 | bufferCh: make(chan *frameworkMetric, bufferSize), 55 | bufferSize: bufferSize, 56 | interval: interval, 57 | stopCh: make(chan struct{}), 58 | isStoppedCh: make(chan struct{}), 59 | } 60 | go recorder.run() 61 | return recorder 62 | } 63 | 64 | // observePluginDurationAsync observes the plugin_execution_duration_seconds metric. 65 | // The metric will be flushed to Prometheus asynchronously. 66 | func (r *metricsRecorder) observePluginDurationAsync(extensionPoint, pluginName string, status *framework.Status, value float64) { 67 | newMetric := &frameworkMetric{ 68 | metric: metrics.PluginExecutionDuration, 69 | labelValues: []string{pluginName, extensionPoint, status.Code().String()}, 70 | value: value, 71 | } 72 | select { 73 | case r.bufferCh <- newMetric: 74 | default: 75 | } 76 | } 77 | 78 | // run flushes buffered metrics into Prometheus every second. 79 | func (r *metricsRecorder) run() { 80 | for { 81 | select { 82 | case <-r.stopCh: 83 | close(r.isStoppedCh) 84 | return 85 | default: 86 | } 87 | r.flushMetrics() 88 | time.Sleep(r.interval) 89 | } 90 | } 91 | 92 | // flushMetrics tries to clean up the bufferCh by reading at most bufferSize metrics. 93 | func (r *metricsRecorder) flushMetrics() { 94 | for i := 0; i < r.bufferSize; i++ { 95 | select { 96 | case m := <-r.bufferCh: 97 | m.metric.WithLabelValues(m.labelValues...).Observe(m.value) 98 | default: 99 | return 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/examples/stateful/stateful.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stateful 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "sync" 23 | 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/klog/v2" 27 | "k8s.io/kubernetes/pkg/scheduler/framework" 28 | ) 29 | 30 | // MultipointExample is an example plugin that is executed at multiple extension points. 31 | // This plugin is stateful. It receives arguments at initialization (NewMultipointPlugin) 32 | // and changes its state when it is executed. 33 | type MultipointExample struct { 34 | executionPoints []string 35 | mu sync.RWMutex 36 | } 37 | 38 | var _ framework.ReservePlugin = &MultipointExample{} 39 | var _ framework.PreBindPlugin = &MultipointExample{} 40 | 41 | // Name is the name of the plug used in Registry and configurations. 42 | const Name = "multipoint-plugin-example" 43 | 44 | // Name returns name of the plugin. It is used in logs, etc. 45 | func (mp *MultipointExample) Name() string { 46 | return Name 47 | } 48 | 49 | // Reserve is the function invoked by the framework at "reserve" extension 50 | // point. In this trivial example, the Reserve method allocates an array of 51 | // strings. 52 | func (mp *MultipointExample) Reserve(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) *framework.Status { 53 | // Reserve is not called concurrently, and so we don't need to lock. 54 | mp.executionPoints = append(mp.executionPoints, "reserve") 55 | return nil 56 | } 57 | 58 | // Unreserve is the function invoked by the framework when any error happens 59 | // during "reserve" extension point or later. In this example, the Unreserve 60 | // method loses its reference to the string slice, allowing it to be garbage 61 | // collected, and thereby "unallocating" the reserved resources. 62 | func (mp *MultipointExample) Unreserve(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) { 63 | // Unlike Reserve, the Unreserve method may be called concurrently since 64 | // there is no guarantee that there will only one unreserve operation at any 65 | // given point in time (for example, during the binding cycle). 66 | mp.mu.Lock() 67 | defer mp.mu.Unlock() 68 | mp.executionPoints = nil 69 | } 70 | 71 | // PreBind is the function invoked by the framework at "prebind" extension 72 | // point. 73 | func (mp *MultipointExample) PreBind(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) *framework.Status { 74 | // PreBind could be called concurrently for different pods. 75 | mp.mu.Lock() 76 | defer mp.mu.Unlock() 77 | mp.executionPoints = append(mp.executionPoints, "pre-bind") 78 | if pod == nil { 79 | return framework.NewStatus(framework.Error, "pod must not be nil") 80 | } 81 | return nil 82 | } 83 | 84 | // New initializes a new plugin and returns it. 85 | func New(config *runtime.Unknown, _ framework.Handle) (framework.Plugin, error) { 86 | if config == nil { 87 | klog.ErrorS(nil, "MultipointExample configuration cannot be empty") 88 | return nil, fmt.Errorf("MultipointExample configuration cannot be empty") 89 | } 90 | mp := MultipointExample{} 91 | return &mp, nil 92 | } 93 | -------------------------------------------------------------------------------- /scheduler/framework/runtime/registry.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package runtime 18 | 19 | import ( 20 | "fmt" 21 | 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/apimachinery/pkg/util/json" 24 | "k8s.io/kubernetes/pkg/scheduler/framework" 25 | plfeature "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" 26 | "sigs.k8s.io/yaml" 27 | ) 28 | 29 | // PluginFactory is a function that builds a plugin. 30 | type PluginFactory = func(configuration runtime.Object, f framework.Handle) (framework.Plugin, error) 31 | 32 | // PluginFactoryWithFts is a function that builds a plugin with certain feature gates. 33 | type PluginFactoryWithFts func(runtime.Object, framework.Handle, plfeature.Features) (framework.Plugin, error) 34 | 35 | // FactoryAdapter can be used to inject feature gates for a plugin that needs 36 | // them when the caller expects the older PluginFactory method. 37 | func FactoryAdapter(fts plfeature.Features, withFts PluginFactoryWithFts) PluginFactory { 38 | return func(plArgs runtime.Object, fh framework.Handle) (framework.Plugin, error) { 39 | return withFts(plArgs, fh, fts) 40 | } 41 | } 42 | 43 | // DecodeInto decodes configuration whose type is *runtime.Unknown to the interface into. 44 | func DecodeInto(obj runtime.Object, into interface{}) error { 45 | if obj == nil { 46 | return nil 47 | } 48 | configuration, ok := obj.(*runtime.Unknown) 49 | if !ok { 50 | return fmt.Errorf("want args of type runtime.Unknown, got %T", obj) 51 | } 52 | if configuration.Raw == nil { 53 | return nil 54 | } 55 | 56 | switch configuration.ContentType { 57 | // If ContentType is empty, it means ContentTypeJSON by default. 58 | case runtime.ContentTypeJSON, "": 59 | return json.Unmarshal(configuration.Raw, into) 60 | case runtime.ContentTypeYAML: 61 | return yaml.Unmarshal(configuration.Raw, into) 62 | default: 63 | return fmt.Errorf("not supported content type %s", configuration.ContentType) 64 | } 65 | } 66 | 67 | // Registry is a collection of all available plugins. The framework uses a 68 | // registry to enable and initialize configured plugins. 69 | // All plugins must be in the registry before initializing the framework. 70 | type Registry map[string]PluginFactory 71 | 72 | // Register adds a new plugin to the registry. If a plugin with the same name 73 | // exists, it returns an error. 74 | func (r Registry) Register(name string, factory PluginFactory) error { 75 | if _, ok := r[name]; ok { 76 | return fmt.Errorf("a plugin named %v already exists", name) 77 | } 78 | r[name] = factory 79 | return nil 80 | } 81 | 82 | // Unregister removes an existing plugin from the registry. If no plugin with 83 | // the provided name exists, it returns an error. 84 | func (r Registry) Unregister(name string) error { 85 | if _, ok := r[name]; !ok { 86 | return fmt.Errorf("no plugin named %v exists", name) 87 | } 88 | delete(r, name) 89 | return nil 90 | } 91 | 92 | // Merge merges the provided registry to the current one. 93 | func (r Registry) Merge(in Registry) error { 94 | for name, factory := range in { 95 | if err := r.Register(name, factory); err != nil { 96 | return err 97 | } 98 | } 99 | return nil 100 | } 101 | -------------------------------------------------------------------------------- /scheduler/framework/cycle_state.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package framework 18 | 19 | import ( 20 | "errors" 21 | "sync" 22 | ) 23 | 24 | var ( 25 | // ErrNotFound is the not found error message. 26 | ErrNotFound = errors.New("not found") 27 | ) 28 | 29 | // StateData is a generic type for arbitrary data stored in CycleState. 30 | type StateData interface { 31 | // Clone is an interface to make a copy of StateData. For performance reasons, 32 | // clone should make shallow copies for members (e.g., slices or maps) that are not 33 | // impacted by PreFilter's optional AddPod/RemovePod methods. 34 | Clone() StateData 35 | } 36 | 37 | // StateKey is the type of keys stored in CycleState. 38 | type StateKey string 39 | 40 | // CycleState provides a mechanism for plugins to store and retrieve arbitrary data. 41 | // StateData stored by one plugin can be read, altered, or deleted by another plugin. 42 | // CycleState does not provide any data protection, as all plugins are assumed to be 43 | // trusted. 44 | type CycleState struct { 45 | mx sync.RWMutex 46 | storage map[StateKey]StateData 47 | // if recordPluginMetrics is true, PluginExecutionDuration will be recorded for this cycle. 48 | recordPluginMetrics bool 49 | } 50 | 51 | // NewCycleState initializes a new CycleState and returns its pointer. 52 | func NewCycleState() *CycleState { 53 | return &CycleState{ 54 | storage: make(map[StateKey]StateData), 55 | } 56 | } 57 | 58 | // ShouldRecordPluginMetrics returns whether PluginExecutionDuration metrics should be recorded. 59 | func (c *CycleState) ShouldRecordPluginMetrics() bool { 60 | if c == nil { 61 | return false 62 | } 63 | return c.recordPluginMetrics 64 | } 65 | 66 | // SetRecordPluginMetrics sets recordPluginMetrics to the given value. 67 | func (c *CycleState) SetRecordPluginMetrics(flag bool) { 68 | if c == nil { 69 | return 70 | } 71 | c.recordPluginMetrics = flag 72 | } 73 | 74 | // Clone creates a copy of CycleState and returns its pointer. Clone returns 75 | // nil if the context being cloned is nil. 76 | func (c *CycleState) Clone() *CycleState { 77 | if c == nil { 78 | return nil 79 | } 80 | copy := NewCycleState() 81 | for k, v := range c.storage { 82 | copy.Write(k, v.Clone()) 83 | } 84 | return copy 85 | } 86 | 87 | // Read retrieves data with the given "key" from CycleState. If the key is not 88 | // present an error is returned. 89 | // This function is thread safe by acquiring an internal lock first. 90 | func (c *CycleState) Read(key StateKey) (StateData, error) { 91 | c.mx.RLock() 92 | defer c.mx.RUnlock() 93 | if v, ok := c.storage[key]; ok { 94 | return v, nil 95 | } 96 | return nil, ErrNotFound 97 | } 98 | 99 | // Write stores the given "val" in CycleState with the given "key". 100 | // This function is thread safe by acquiring an internal lock first. 101 | func (c *CycleState) Write(key StateKey, val StateData) { 102 | c.mx.Lock() 103 | c.storage[key] = val 104 | c.mx.Unlock() 105 | } 106 | 107 | // Delete deletes data with the given key from CycleState. 108 | // This function is thread safe by acquiring an internal lock first. 109 | func (c *CycleState) Delete(key StateKey) { 110 | c.mx.Lock() 111 | delete(c.storage, key) 112 | c.mx.Unlock() 113 | } 114 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/helper/spread.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helper 18 | 19 | import ( 20 | appsv1 "k8s.io/api/apps/v1" 21 | v1 "k8s.io/api/core/v1" 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | "k8s.io/apimachinery/pkg/labels" 24 | "k8s.io/apimachinery/pkg/runtime/schema" 25 | appslisters "k8s.io/client-go/listers/apps/v1" 26 | corelisters "k8s.io/client-go/listers/core/v1" 27 | ) 28 | 29 | var ( 30 | rcKind = v1.SchemeGroupVersion.WithKind("ReplicationController") 31 | rsKind = appsv1.SchemeGroupVersion.WithKind("ReplicaSet") 32 | ssKind = appsv1.SchemeGroupVersion.WithKind("StatefulSet") 33 | ) 34 | 35 | // DefaultSelector returns a selector deduced from the Services, Replication 36 | // Controllers, Replica Sets, and Stateful Sets matching the given pod. 37 | func DefaultSelector( 38 | pod *v1.Pod, 39 | sl corelisters.ServiceLister, 40 | cl corelisters.ReplicationControllerLister, 41 | rsl appslisters.ReplicaSetLister, 42 | ssl appslisters.StatefulSetLister, 43 | ) labels.Selector { 44 | labelSet := make(labels.Set) 45 | // Since services, RCs, RSs and SSs match the pod, they won't have conflicting 46 | // labels. Merging is safe. 47 | 48 | if services, err := GetPodServices(sl, pod); err == nil { 49 | for _, service := range services { 50 | labelSet = labels.Merge(labelSet, service.Spec.Selector) 51 | } 52 | } 53 | selector := labelSet.AsSelector() 54 | 55 | owner := metav1.GetControllerOfNoCopy(pod) 56 | if owner == nil { 57 | return selector 58 | } 59 | 60 | gv, err := schema.ParseGroupVersion(owner.APIVersion) 61 | if err != nil { 62 | return selector 63 | } 64 | 65 | gvk := gv.WithKind(owner.Kind) 66 | switch gvk { 67 | case rcKind: 68 | if rc, err := cl.ReplicationControllers(pod.Namespace).Get(owner.Name); err == nil { 69 | labelSet = labels.Merge(labelSet, rc.Spec.Selector) 70 | selector = labelSet.AsSelector() 71 | } 72 | case rsKind: 73 | if rs, err := rsl.ReplicaSets(pod.Namespace).Get(owner.Name); err == nil { 74 | if other, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector); err == nil { 75 | if r, ok := other.Requirements(); ok { 76 | selector = selector.Add(r...) 77 | } 78 | } 79 | } 80 | case ssKind: 81 | if ss, err := ssl.StatefulSets(pod.Namespace).Get(owner.Name); err == nil { 82 | if other, err := metav1.LabelSelectorAsSelector(ss.Spec.Selector); err == nil { 83 | if r, ok := other.Requirements(); ok { 84 | selector = selector.Add(r...) 85 | } 86 | } 87 | } 88 | default: 89 | // Not owned by a supported controller. 90 | } 91 | 92 | return selector 93 | } 94 | 95 | // GetPodServices gets the services that have the selector that match the labels on the given pod. 96 | func GetPodServices(sl corelisters.ServiceLister, pod *v1.Pod) ([]*v1.Service, error) { 97 | allServices, err := sl.Services(pod.Namespace).List(labels.Everything()) 98 | if err != nil { 99 | return nil, err 100 | } 101 | 102 | var services []*v1.Service 103 | for i := range allServices { 104 | service := allServices[i] 105 | if service.Spec.Selector == nil { 106 | // services with nil selectors match nothing, not everything. 107 | continue 108 | } 109 | selector := labels.Set(service.Spec.Selector).AsSelectorPreValidated() 110 | if selector.Matches(labels.Set(pod.Labels)) { 111 | services = append(services, service) 112 | } 113 | } 114 | 115 | return services, nil 116 | } 117 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta2/conversion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta2 18 | 19 | import ( 20 | "fmt" 21 | "sync" 22 | 23 | "k8s.io/apimachinery/pkg/conversion" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 26 | "k8s.io/kube-scheduler/config/v1beta2" 27 | "k8s.io/kubernetes/pkg/scheduler/apis/config" 28 | ) 29 | 30 | var ( 31 | // pluginArgConversionScheme is a scheme with internal and v1beta2 registered, 32 | // used for defaulting/converting typed PluginConfig Args. 33 | // Access via getPluginArgConversionScheme() 34 | pluginArgConversionScheme *runtime.Scheme 35 | initPluginArgConversionScheme sync.Once 36 | ) 37 | 38 | func GetPluginArgConversionScheme() *runtime.Scheme { 39 | initPluginArgConversionScheme.Do(func() { 40 | // set up the scheme used for plugin arg conversion 41 | pluginArgConversionScheme = runtime.NewScheme() 42 | utilruntime.Must(AddToScheme(pluginArgConversionScheme)) 43 | utilruntime.Must(config.AddToScheme(pluginArgConversionScheme)) 44 | }) 45 | return pluginArgConversionScheme 46 | } 47 | 48 | func Convert_v1beta2_KubeSchedulerConfiguration_To_config_KubeSchedulerConfiguration(in *v1beta2.KubeSchedulerConfiguration, out *config.KubeSchedulerConfiguration, s conversion.Scope) error { 49 | if err := autoConvert_v1beta2_KubeSchedulerConfiguration_To_config_KubeSchedulerConfiguration(in, out, s); err != nil { 50 | return err 51 | } 52 | return convertToInternalPluginConfigArgs(out) 53 | } 54 | 55 | // convertToInternalPluginConfigArgs converts PluginConfig#Args into internal 56 | // types using a scheme, after applying defaults. 57 | func convertToInternalPluginConfigArgs(out *config.KubeSchedulerConfiguration) error { 58 | scheme := GetPluginArgConversionScheme() 59 | for i := range out.Profiles { 60 | prof := &out.Profiles[i] 61 | for j := range prof.PluginConfig { 62 | args := prof.PluginConfig[j].Args 63 | if args == nil { 64 | continue 65 | } 66 | if _, isUnknown := args.(*runtime.Unknown); isUnknown { 67 | continue 68 | } 69 | internalArgs, err := scheme.ConvertToVersion(args, config.SchemeGroupVersion) 70 | if err != nil { 71 | return fmt.Errorf("converting .Profiles[%d].PluginConfig[%d].Args into internal type: %w", i, j, err) 72 | } 73 | prof.PluginConfig[j].Args = internalArgs 74 | } 75 | } 76 | return nil 77 | } 78 | 79 | func Convert_config_KubeSchedulerConfiguration_To_v1beta2_KubeSchedulerConfiguration(in *config.KubeSchedulerConfiguration, out *v1beta2.KubeSchedulerConfiguration, s conversion.Scope) error { 80 | if err := autoConvert_config_KubeSchedulerConfiguration_To_v1beta2_KubeSchedulerConfiguration(in, out, s); err != nil { 81 | return err 82 | } 83 | return convertToExternalPluginConfigArgs(out) 84 | } 85 | 86 | // convertToExternalPluginConfigArgs converts PluginConfig#Args into 87 | // external (versioned) types using a scheme. 88 | func convertToExternalPluginConfigArgs(out *v1beta2.KubeSchedulerConfiguration) error { 89 | scheme := GetPluginArgConversionScheme() 90 | for i := range out.Profiles { 91 | for j := range out.Profiles[i].PluginConfig { 92 | args := out.Profiles[i].PluginConfig[j].Args 93 | if args.Object == nil { 94 | continue 95 | } 96 | if _, isUnknown := args.Object.(*runtime.Unknown); isUnknown { 97 | continue 98 | } 99 | externalArgs, err := scheme.ConvertToVersion(args.Object, SchemeGroupVersion) 100 | if err != nil { 101 | return err 102 | } 103 | out.Profiles[i].PluginConfig[j].Args.Object = externalArgs 104 | } 105 | } 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /scheduler/apis/config/v1beta3/conversion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta3 18 | 19 | import ( 20 | "fmt" 21 | "sync" 22 | 23 | "k8s.io/apimachinery/pkg/conversion" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 26 | "k8s.io/kube-scheduler/config/v1beta3" 27 | "k8s.io/kubernetes/pkg/scheduler/apis/config" 28 | ) 29 | 30 | var ( 31 | // pluginArgConversionScheme is a scheme with internal and v1beta3 registered, 32 | // used for defaulting/converting typed PluginConfig Args. 33 | // Access via getPluginArgConversionScheme() 34 | pluginArgConversionScheme *runtime.Scheme 35 | initPluginArgConversionScheme sync.Once 36 | ) 37 | 38 | func GetPluginArgConversionScheme() *runtime.Scheme { 39 | initPluginArgConversionScheme.Do(func() { 40 | // set up the scheme used for plugin arg conversion 41 | pluginArgConversionScheme = runtime.NewScheme() 42 | utilruntime.Must(AddToScheme(pluginArgConversionScheme)) 43 | utilruntime.Must(config.AddToScheme(pluginArgConversionScheme)) 44 | }) 45 | return pluginArgConversionScheme 46 | } 47 | 48 | func Convert_v1beta3_KubeSchedulerConfiguration_To_config_KubeSchedulerConfiguration(in *v1beta3.KubeSchedulerConfiguration, out *config.KubeSchedulerConfiguration, s conversion.Scope) error { 49 | if err := autoConvert_v1beta3_KubeSchedulerConfiguration_To_config_KubeSchedulerConfiguration(in, out, s); err != nil { 50 | return err 51 | } 52 | return convertToInternalPluginConfigArgs(out) 53 | } 54 | 55 | // convertToInternalPluginConfigArgs converts PluginConfig#Args into internal 56 | // types using a scheme, after applying defaults. 57 | func convertToInternalPluginConfigArgs(out *config.KubeSchedulerConfiguration) error { 58 | scheme := GetPluginArgConversionScheme() 59 | for i := range out.Profiles { 60 | prof := &out.Profiles[i] 61 | for j := range prof.PluginConfig { 62 | args := prof.PluginConfig[j].Args 63 | if args == nil { 64 | continue 65 | } 66 | if _, isUnknown := args.(*runtime.Unknown); isUnknown { 67 | continue 68 | } 69 | internalArgs, err := scheme.ConvertToVersion(args, config.SchemeGroupVersion) 70 | if err != nil { 71 | return fmt.Errorf("converting .Profiles[%d].PluginConfig[%d].Args into internal type: %w", i, j, err) 72 | } 73 | prof.PluginConfig[j].Args = internalArgs 74 | } 75 | } 76 | return nil 77 | } 78 | 79 | func Convert_config_KubeSchedulerConfiguration_To_v1beta3_KubeSchedulerConfiguration(in *config.KubeSchedulerConfiguration, out *v1beta3.KubeSchedulerConfiguration, s conversion.Scope) error { 80 | if err := autoConvert_config_KubeSchedulerConfiguration_To_v1beta3_KubeSchedulerConfiguration(in, out, s); err != nil { 81 | return err 82 | } 83 | return convertToExternalPluginConfigArgs(out) 84 | } 85 | 86 | // convertToExternalPluginConfigArgs converts PluginConfig#Args into 87 | // external (versioned) types using a scheme. 88 | func convertToExternalPluginConfigArgs(out *v1beta3.KubeSchedulerConfiguration) error { 89 | scheme := GetPluginArgConversionScheme() 90 | for i := range out.Profiles { 91 | for j := range out.Profiles[i].PluginConfig { 92 | args := out.Profiles[i].PluginConfig[j].Args 93 | if args.Object == nil { 94 | continue 95 | } 96 | if _, isUnknown := args.Object.(*runtime.Unknown); isUnknown { 97 | continue 98 | } 99 | externalArgs, err := scheme.ConvertToVersion(args.Object, SchemeGroupVersion) 100 | if err != nil { 101 | return err 102 | } 103 | out.Profiles[i].PluginConfig[j].Args.Object = externalArgs 104 | } 105 | } 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /scheduler/internal/cache/debugger/comparer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package debugger 18 | 19 | import ( 20 | "sort" 21 | "strings" 22 | 23 | "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/labels" 25 | corelisters "k8s.io/client-go/listers/core/v1" 26 | "k8s.io/klog/v2" 27 | "k8s.io/kubernetes/pkg/scheduler/framework" 28 | internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" 29 | internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" 30 | ) 31 | 32 | // CacheComparer is an implementation of the Scheduler's cache comparer. 33 | type CacheComparer struct { 34 | NodeLister corelisters.NodeLister 35 | PodLister corelisters.PodLister 36 | Cache internalcache.Cache 37 | PodQueue internalqueue.SchedulingQueue 38 | } 39 | 40 | // Compare compares the nodes and pods of NodeLister with Cache.Snapshot. 41 | func (c *CacheComparer) Compare() error { 42 | klog.V(3).InfoS("Cache comparer started") 43 | defer klog.V(3).InfoS("Cache comparer finished") 44 | 45 | nodes, err := c.NodeLister.List(labels.Everything()) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | pods, err := c.PodLister.List(labels.Everything()) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | dump := c.Cache.Dump() 56 | 57 | pendingPods := c.PodQueue.PendingPods() 58 | 59 | if missed, redundant := c.CompareNodes(nodes, dump.Nodes); len(missed)+len(redundant) != 0 { 60 | klog.InfoS("Cache mismatch", "missedNodes", missed, "redundantNodes", redundant) 61 | } 62 | 63 | if missed, redundant := c.ComparePods(pods, pendingPods, dump.Nodes); len(missed)+len(redundant) != 0 { 64 | klog.InfoS("Cache mismatch", "missedPods", missed, "redundantPods", redundant) 65 | } 66 | 67 | return nil 68 | } 69 | 70 | // CompareNodes compares actual nodes with cached nodes. 71 | func (c *CacheComparer) CompareNodes(nodes []*v1.Node, nodeinfos map[string]*framework.NodeInfo) (missed, redundant []string) { 72 | actual := []string{} 73 | for _, node := range nodes { 74 | actual = append(actual, node.Name) 75 | } 76 | 77 | cached := []string{} 78 | for nodeName := range nodeinfos { 79 | cached = append(cached, nodeName) 80 | } 81 | 82 | return compareStrings(actual, cached) 83 | } 84 | 85 | // ComparePods compares actual pods with cached pods. 86 | func (c *CacheComparer) ComparePods(pods, waitingPods []*v1.Pod, nodeinfos map[string]*framework.NodeInfo) (missed, redundant []string) { 87 | actual := []string{} 88 | for _, pod := range pods { 89 | actual = append(actual, string(pod.UID)) 90 | } 91 | 92 | cached := []string{} 93 | for _, nodeinfo := range nodeinfos { 94 | for _, p := range nodeinfo.Pods { 95 | cached = append(cached, string(p.Pod.UID)) 96 | } 97 | } 98 | for _, pod := range waitingPods { 99 | cached = append(cached, string(pod.UID)) 100 | } 101 | 102 | return compareStrings(actual, cached) 103 | } 104 | 105 | func compareStrings(actual, cached []string) (missed, redundant []string) { 106 | missed, redundant = []string{}, []string{} 107 | 108 | sort.Strings(actual) 109 | sort.Strings(cached) 110 | 111 | compare := func(i, j int) int { 112 | if i == len(actual) { 113 | return 1 114 | } else if j == len(cached) { 115 | return -1 116 | } 117 | return strings.Compare(actual[i], cached[j]) 118 | } 119 | 120 | for i, j := 0, 0; i < len(actual) || j < len(cached); { 121 | switch compare(i, j) { 122 | case 0: 123 | i++ 124 | j++ 125 | case -1: 126 | missed = append(missed, actual[i]) 127 | i++ 128 | case 1: 129 | redundant = append(redundant, cached[j]) 130 | j++ 131 | } 132 | } 133 | 134 | return 135 | } 136 | -------------------------------------------------------------------------------- /scheduler/profile/profile.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package profile holds the definition of a scheduling Profile. 18 | package profile 19 | 20 | import ( 21 | "errors" 22 | "fmt" 23 | "github.com/google/go-cmp/cmp" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/client-go/kubernetes/scheme" 26 | "k8s.io/client-go/tools/events" 27 | "k8s.io/kubernetes/pkg/scheduler/apis/config" 28 | "k8s.io/kubernetes/pkg/scheduler/framework" 29 | frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 30 | ) 31 | 32 | // RecorderFactory builds an EventRecorder for a given scheduler name. 33 | type RecorderFactory func(string) events.EventRecorder 34 | 35 | // newProfile builds a Profile for the given configuration. 36 | func newProfile(cfg config.KubeSchedulerProfile, r frameworkruntime.Registry, recorderFact RecorderFactory, 37 | opts ...frameworkruntime.Option) (framework.Framework, error) { 38 | recorder := recorderFact(cfg.SchedulerName) 39 | opts = append(opts, frameworkruntime.WithEventRecorder(recorder)) 40 | fwk, err := frameworkruntime.NewFramework(r, &cfg, opts...) 41 | if err != nil { 42 | return nil, err 43 | } 44 | return fwk, nil 45 | } 46 | 47 | // Map holds frameworks indexed by scheduler name. 48 | type Map map[string]framework.Framework 49 | 50 | // NewMap builds the frameworks given by the configuration, indexed by name. 51 | func NewMap(cfgs []config.KubeSchedulerProfile, r frameworkruntime.Registry, recorderFact RecorderFactory, 52 | opts ...frameworkruntime.Option) (Map, error) { 53 | m := make(Map) 54 | v := cfgValidator{m: m} 55 | 56 | for _, cfg := range cfgs { 57 | p, err := newProfile(cfg, r, recorderFact, opts...) 58 | if err != nil { 59 | return nil, fmt.Errorf("creating profile for scheduler name %s: %v", cfg.SchedulerName, err) 60 | } 61 | if err := v.validate(cfg, p); err != nil { 62 | return nil, err 63 | } 64 | m[cfg.SchedulerName] = p 65 | } 66 | return m, nil 67 | } 68 | 69 | // HandlesSchedulerName returns whether a profile handles the given scheduler name. 70 | func (m Map) HandlesSchedulerName(name string) bool { 71 | _, ok := m[name] 72 | return ok 73 | } 74 | 75 | // NewRecorderFactory returns a RecorderFactory for the broadcaster. 76 | func NewRecorderFactory(b events.EventBroadcaster) RecorderFactory { 77 | return func(name string) events.EventRecorder { 78 | return b.NewRecorder(scheme.Scheme, name) 79 | } 80 | } 81 | 82 | type cfgValidator struct { 83 | m Map 84 | queueSort string 85 | queueSortArgs runtime.Object 86 | } 87 | 88 | func (v *cfgValidator) validate(cfg config.KubeSchedulerProfile, f framework.Framework) error { 89 | if len(f.ProfileName()) == 0 { 90 | return errors.New("scheduler name is needed") 91 | } 92 | if cfg.Plugins == nil { 93 | return fmt.Errorf("plugins required for profile with scheduler name %q", f.ProfileName()) 94 | } 95 | if v.m[f.ProfileName()] != nil { 96 | return fmt.Errorf("duplicate profile with scheduler name %q", f.ProfileName()) 97 | } 98 | 99 | queueSort := f.ListPlugins().QueueSort.Enabled[0].Name 100 | var queueSortArgs runtime.Object 101 | for _, plCfg := range cfg.PluginConfig { 102 | if plCfg.Name == queueSort { 103 | queueSortArgs = plCfg.Args 104 | } 105 | } 106 | if len(v.queueSort) == 0 { 107 | v.queueSort = queueSort 108 | v.queueSortArgs = queueSortArgs 109 | return nil 110 | } 111 | if v.queueSort != queueSort { 112 | return fmt.Errorf("different queue sort plugins for profile %q: %q, first: %q", cfg.SchedulerName, queueSort, v.queueSort) 113 | } 114 | if !cmp.Equal(v.queueSortArgs, queueSortArgs) { 115 | return fmt.Errorf("different queue sort plugin args for profile %q", cfg.SchedulerName) 116 | } 117 | return nil 118 | } 119 | -------------------------------------------------------------------------------- /scheduler/util/pod_resources_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/api/resource" 26 | ) 27 | 28 | func TestGetNonZeroRequest(t *testing.T) { 29 | tests := []struct { 30 | name string 31 | requests v1.ResourceList 32 | expectedCPU int64 33 | expectedMemory int64 34 | }{ 35 | { 36 | "cpu_and_memory_not_found", 37 | v1.ResourceList{}, 38 | DefaultMilliCPURequest, 39 | DefaultMemoryRequest, 40 | }, 41 | { 42 | "only_cpu_exist", 43 | v1.ResourceList{ 44 | v1.ResourceCPU: resource.MustParse("200m"), 45 | }, 46 | 200, 47 | DefaultMemoryRequest, 48 | }, 49 | { 50 | "only_memory_exist", 51 | v1.ResourceList{ 52 | v1.ResourceMemory: resource.MustParse("400Mi"), 53 | }, 54 | DefaultMilliCPURequest, 55 | 400 * 1024 * 1024, 56 | }, 57 | { 58 | "cpu_memory_exist", 59 | v1.ResourceList{ 60 | v1.ResourceCPU: resource.MustParse("200m"), 61 | v1.ResourceMemory: resource.MustParse("400Mi"), 62 | }, 63 | 200, 64 | 400 * 1024 * 1024, 65 | }, 66 | } 67 | 68 | for _, test := range tests { 69 | t.Run(test.name, func(t *testing.T) { 70 | realCPU, realMemory := GetNonzeroRequests(&test.requests) 71 | assert.EqualValuesf(t, test.expectedCPU, realCPU, "Failed to test: %s", test.name) 72 | assert.EqualValuesf(t, test.expectedMemory, realMemory, "Failed to test: %s", test.name) 73 | }) 74 | } 75 | } 76 | 77 | func TestGetRequestForResource(t *testing.T) { 78 | tests := []struct { 79 | name string 80 | requests v1.ResourceList 81 | resource v1.ResourceName 82 | expectedQuantity int64 83 | nonZero bool 84 | }{ 85 | { 86 | "extended_resource_not_found", 87 | v1.ResourceList{}, 88 | v1.ResourceName("intel.com/foo"), 89 | 0, 90 | true, 91 | }, 92 | { 93 | "extended_resource_found", 94 | v1.ResourceList{ 95 | v1.ResourceName("intel.com/foo"): resource.MustParse("4"), 96 | }, 97 | v1.ResourceName("intel.com/foo"), 98 | 4, 99 | true, 100 | }, 101 | { 102 | "cpu_not_found", 103 | v1.ResourceList{}, 104 | v1.ResourceCPU, 105 | DefaultMilliCPURequest, 106 | true, 107 | }, 108 | { 109 | "memory_not_found", 110 | v1.ResourceList{}, 111 | v1.ResourceMemory, 112 | DefaultMemoryRequest, 113 | true, 114 | }, 115 | { 116 | "cpu_exist", 117 | v1.ResourceList{ 118 | v1.ResourceCPU: resource.MustParse("200m"), 119 | }, 120 | v1.ResourceCPU, 121 | 200, 122 | true, 123 | }, 124 | { 125 | "memory_exist", 126 | v1.ResourceList{ 127 | v1.ResourceMemory: resource.MustParse("400Mi"), 128 | }, 129 | v1.ResourceMemory, 130 | 400 * 1024 * 1024, 131 | true, 132 | }, 133 | { 134 | "ephemeralStorage_exist", 135 | v1.ResourceList{ 136 | v1.ResourceEphemeralStorage: resource.MustParse("400Mi"), 137 | }, 138 | v1.ResourceEphemeralStorage, 139 | 400 * 1024 * 1024, 140 | true, 141 | }, 142 | { 143 | "ephemeralStorage_not_found", 144 | v1.ResourceList{}, 145 | v1.ResourceEphemeralStorage, 146 | 0, 147 | true, 148 | }, 149 | { 150 | "cpu_not_found, useRequested is true", 151 | v1.ResourceList{}, 152 | v1.ResourceCPU, 153 | 0, 154 | false, 155 | }, 156 | { 157 | "memory_not_found, useRequested is true", 158 | v1.ResourceList{}, 159 | v1.ResourceMemory, 160 | 0, 161 | false, 162 | }, 163 | } 164 | 165 | for _, test := range tests { 166 | t.Run(test.name, func(t *testing.T) { 167 | realQuantity := GetRequestForResource(test.resource, &test.requests, test.nonZero) 168 | assert.EqualValuesf(t, test.expectedQuantity, realQuantity, "Failed to test: %s", test.name) 169 | }) 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/registry.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package plugins 18 | 19 | import ( 20 | "k8s.io/apiserver/pkg/util/feature" 21 | "k8s.io/kubernetes/pkg/features" 22 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" 23 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpreemption" 24 | plfeature "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" 25 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality" 26 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" 27 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" 28 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename" 29 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" 30 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" 31 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable" 32 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits" 33 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" 34 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" 35 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/selectorspread" 36 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" 37 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding" 38 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumerestrictions" 39 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumezone" 40 | "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 41 | ) 42 | 43 | // NewInTreeRegistry builds the registry with all the in-tree plugins. 44 | // A scheduler that runs out of tree plugins can register additional plugins 45 | // through the WithFrameworkOutOfTreeRegistry option. 46 | func NewInTreeRegistry() runtime.Registry { 47 | fts := plfeature.Features{ 48 | EnablePodAffinityNamespaceSelector: feature.DefaultFeatureGate.Enabled(features.PodAffinityNamespaceSelector), 49 | EnablePodDisruptionBudget: feature.DefaultFeatureGate.Enabled(features.PodDisruptionBudget), 50 | EnablePodOverhead: feature.DefaultFeatureGate.Enabled(features.PodOverhead), 51 | EnableReadWriteOncePod: feature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod), 52 | EnableVolumeCapacityPriority: feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority), 53 | EnableCSIStorageCapacity: feature.DefaultFeatureGate.Enabled(features.CSIStorageCapacity), 54 | } 55 | 56 | return runtime.Registry{ 57 | selectorspread.Name: selectorspread.New, 58 | imagelocality.Name: imagelocality.New, 59 | tainttoleration.Name: tainttoleration.New, 60 | nodename.Name: nodename.New, 61 | nodeports.Name: nodeports.New, 62 | nodeaffinity.Name: nodeaffinity.New, 63 | podtopologyspread.Name: podtopologyspread.New, 64 | nodeunschedulable.Name: nodeunschedulable.New, 65 | noderesources.FitName: runtime.FactoryAdapter(fts, noderesources.NewFit), 66 | noderesources.BalancedAllocationName: runtime.FactoryAdapter(fts, noderesources.NewBalancedAllocation), 67 | volumebinding.Name: runtime.FactoryAdapter(fts, volumebinding.New), 68 | volumerestrictions.Name: runtime.FactoryAdapter(fts, volumerestrictions.New), 69 | volumezone.Name: volumezone.New, 70 | nodevolumelimits.CSIName: runtime.FactoryAdapter(fts, nodevolumelimits.NewCSI), 71 | nodevolumelimits.EBSName: runtime.FactoryAdapter(fts, nodevolumelimits.NewEBS), 72 | nodevolumelimits.GCEPDName: runtime.FactoryAdapter(fts, nodevolumelimits.NewGCEPD), 73 | nodevolumelimits.AzureDiskName: runtime.FactoryAdapter(fts, nodevolumelimits.NewAzureDisk), 74 | nodevolumelimits.CinderName: runtime.FactoryAdapter(fts, nodevolumelimits.NewCinder), 75 | interpodaffinity.Name: runtime.FactoryAdapter(fts, interpodaffinity.New), 76 | queuesort.Name: queuesort.New, 77 | defaultbinder.Name: defaultbinder.New, 78 | defaultpreemption.Name: runtime.FactoryAdapter(fts, defaultpreemption.New), 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /scheduler/internal/cache/node_tree.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cache 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | utilnode "k8s.io/component-helpers/node/topology" 25 | "k8s.io/klog/v2" 26 | ) 27 | 28 | // nodeTree is a tree-like data structure that holds node names in each zone. Zone names are 29 | // keys to "NodeTree.tree" and values of "NodeTree.tree" are arrays of node names. 30 | // NodeTree is NOT thread-safe, any concurrent updates/reads from it must be synchronized by the caller. 31 | // It is used only by schedulerCache, and should stay as such. 32 | type nodeTree struct { 33 | tree map[string][]string // a map from zone (region-zone) to an array of nodes in the zone. 34 | zones []string // a list of all the zones in the tree (keys) 35 | numNodes int 36 | } 37 | 38 | // newNodeTree creates a NodeTree from nodes. 39 | func newNodeTree(nodes []*v1.Node) *nodeTree { 40 | nt := &nodeTree{ 41 | tree: make(map[string][]string), 42 | } 43 | for _, n := range nodes { 44 | nt.addNode(n) 45 | } 46 | return nt 47 | } 48 | 49 | // addNode adds a node and its corresponding zone to the tree. If the zone already exists, the node 50 | // is added to the array of nodes in that zone. 51 | func (nt *nodeTree) addNode(n *v1.Node) { 52 | zone := utilnode.GetZoneKey(n) 53 | if na, ok := nt.tree[zone]; ok { 54 | for _, nodeName := range na { 55 | if nodeName == n.Name { 56 | klog.InfoS("Node already exists in the NodeTree", "node", klog.KObj(n)) 57 | return 58 | } 59 | } 60 | nt.tree[zone] = append(na, n.Name) 61 | } else { 62 | nt.zones = append(nt.zones, zone) 63 | nt.tree[zone] = []string{n.Name} 64 | } 65 | klog.V(2).InfoS("Added node in listed group to NodeTree", "node", klog.KObj(n), "zone", zone) 66 | nt.numNodes++ 67 | } 68 | 69 | // removeNode removes a node from the NodeTree. 70 | func (nt *nodeTree) removeNode(n *v1.Node) error { 71 | zone := utilnode.GetZoneKey(n) 72 | if na, ok := nt.tree[zone]; ok { 73 | for i, nodeName := range na { 74 | if nodeName == n.Name { 75 | nt.tree[zone] = append(na[:i], na[i+1:]...) 76 | if len(nt.tree[zone]) == 0 { 77 | nt.removeZone(zone) 78 | } 79 | klog.V(2).InfoS("Removed node in listed group from NodeTree", "node", klog.KObj(n), "zone", zone) 80 | nt.numNodes-- 81 | return nil 82 | } 83 | } 84 | } 85 | klog.ErrorS(nil, "Node in listed group was not found", "node", klog.KObj(n), "zone", zone) 86 | return fmt.Errorf("node %q in group %q was not found", n.Name, zone) 87 | } 88 | 89 | // removeZone removes a zone from tree. 90 | // This function must be called while writer locks are hold. 91 | func (nt *nodeTree) removeZone(zone string) { 92 | delete(nt.tree, zone) 93 | for i, z := range nt.zones { 94 | if z == zone { 95 | nt.zones = append(nt.zones[:i], nt.zones[i+1:]...) 96 | return 97 | } 98 | } 99 | } 100 | 101 | // updateNode updates a node in the NodeTree. 102 | func (nt *nodeTree) updateNode(old, new *v1.Node) { 103 | var oldZone string 104 | if old != nil { 105 | oldZone = utilnode.GetZoneKey(old) 106 | } 107 | newZone := utilnode.GetZoneKey(new) 108 | // If the zone ID of the node has not changed, we don't need to do anything. Name of the node 109 | // cannot be changed in an update. 110 | if oldZone == newZone { 111 | return 112 | } 113 | nt.removeNode(old) // No error checking. We ignore whether the old node exists or not. 114 | nt.addNode(new) 115 | } 116 | 117 | // list returns the list of names of the node. NodeTree iterates over zones and in each zone iterates 118 | // over nodes in a round robin fashion. 119 | func (nt *nodeTree) list() ([]string, error) { 120 | if len(nt.zones) == 0 { 121 | return nil, nil 122 | } 123 | nodesList := make([]string, 0, nt.numNodes) 124 | numExhaustedZones := 0 125 | nodeIndex := 0 126 | for len(nodesList) < nt.numNodes { 127 | if numExhaustedZones >= len(nt.zones) { // all zones are exhausted. 128 | return nodesList, errors.New("all zones exhausted before reaching count of nodes expected") 129 | } 130 | for zoneIndex := 0; zoneIndex < len(nt.zones); zoneIndex++ { 131 | na := nt.tree[nt.zones[zoneIndex]] 132 | if nodeIndex >= len(na) { // If the zone is exhausted, continue 133 | if nodeIndex == len(na) { // If it is the first time the zone is exhausted 134 | numExhaustedZones++ 135 | } 136 | continue 137 | } 138 | nodesList = append(nodesList, na[nodeIndex]) 139 | } 140 | nodeIndex++ 141 | } 142 | return nodesList, nil 143 | } 144 | -------------------------------------------------------------------------------- /scheduler/internal/cache/snapshot_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cache 18 | 19 | import ( 20 | "fmt" 21 | "reflect" 22 | "testing" 23 | 24 | "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/util/sets" 27 | "k8s.io/kubernetes/pkg/scheduler/framework" 28 | ) 29 | 30 | const mb int64 = 1024 * 1024 31 | 32 | func TestGetNodeImageStates(t *testing.T) { 33 | tests := []struct { 34 | node *v1.Node 35 | imageExistenceMap map[string]sets.String 36 | expected map[string]*framework.ImageStateSummary 37 | }{ 38 | { 39 | node: &v1.Node{ 40 | ObjectMeta: metav1.ObjectMeta{Name: "node-0"}, 41 | Status: v1.NodeStatus{ 42 | Images: []v1.ContainerImage{ 43 | { 44 | Names: []string{ 45 | "gcr.io/10:v1", 46 | }, 47 | SizeBytes: int64(10 * mb), 48 | }, 49 | { 50 | Names: []string{ 51 | "gcr.io/200:v1", 52 | }, 53 | SizeBytes: int64(200 * mb), 54 | }, 55 | }, 56 | }, 57 | }, 58 | imageExistenceMap: map[string]sets.String{ 59 | "gcr.io/10:v1": sets.NewString("node-0", "node-1"), 60 | "gcr.io/200:v1": sets.NewString("node-0"), 61 | }, 62 | expected: map[string]*framework.ImageStateSummary{ 63 | "gcr.io/10:v1": { 64 | Size: int64(10 * mb), 65 | NumNodes: 2, 66 | }, 67 | "gcr.io/200:v1": { 68 | Size: int64(200 * mb), 69 | NumNodes: 1, 70 | }, 71 | }, 72 | }, 73 | { 74 | node: &v1.Node{ 75 | ObjectMeta: metav1.ObjectMeta{Name: "node-0"}, 76 | Status: v1.NodeStatus{}, 77 | }, 78 | imageExistenceMap: map[string]sets.String{ 79 | "gcr.io/10:v1": sets.NewString("node-1"), 80 | "gcr.io/200:v1": sets.NewString(), 81 | }, 82 | expected: map[string]*framework.ImageStateSummary{}, 83 | }, 84 | } 85 | 86 | for i, test := range tests { 87 | t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { 88 | imageStates := getNodeImageStates(test.node, test.imageExistenceMap) 89 | if !reflect.DeepEqual(test.expected, imageStates) { 90 | t.Errorf("expected: %#v, got: %#v", test.expected, imageStates) 91 | } 92 | }) 93 | } 94 | } 95 | 96 | func TestCreateImageExistenceMap(t *testing.T) { 97 | tests := []struct { 98 | nodes []*v1.Node 99 | expected map[string]sets.String 100 | }{ 101 | { 102 | nodes: []*v1.Node{ 103 | { 104 | ObjectMeta: metav1.ObjectMeta{Name: "node-0"}, 105 | Status: v1.NodeStatus{ 106 | Images: []v1.ContainerImage{ 107 | { 108 | Names: []string{ 109 | "gcr.io/10:v1", 110 | }, 111 | SizeBytes: int64(10 * mb), 112 | }, 113 | }, 114 | }, 115 | }, 116 | { 117 | ObjectMeta: metav1.ObjectMeta{Name: "node-1"}, 118 | Status: v1.NodeStatus{ 119 | Images: []v1.ContainerImage{ 120 | { 121 | Names: []string{ 122 | "gcr.io/10:v1", 123 | }, 124 | SizeBytes: int64(10 * mb), 125 | }, 126 | { 127 | Names: []string{ 128 | "gcr.io/200:v1", 129 | }, 130 | SizeBytes: int64(200 * mb), 131 | }, 132 | }, 133 | }, 134 | }, 135 | }, 136 | expected: map[string]sets.String{ 137 | "gcr.io/10:v1": sets.NewString("node-0", "node-1"), 138 | "gcr.io/200:v1": sets.NewString("node-1"), 139 | }, 140 | }, 141 | { 142 | nodes: []*v1.Node{ 143 | { 144 | ObjectMeta: metav1.ObjectMeta{Name: "node-0"}, 145 | Status: v1.NodeStatus{}, 146 | }, 147 | { 148 | ObjectMeta: metav1.ObjectMeta{Name: "node-1"}, 149 | Status: v1.NodeStatus{ 150 | Images: []v1.ContainerImage{ 151 | { 152 | Names: []string{ 153 | "gcr.io/10:v1", 154 | }, 155 | SizeBytes: int64(10 * mb), 156 | }, 157 | { 158 | Names: []string{ 159 | "gcr.io/200:v1", 160 | }, 161 | SizeBytes: int64(200 * mb), 162 | }, 163 | }, 164 | }, 165 | }, 166 | }, 167 | expected: map[string]sets.String{ 168 | "gcr.io/10:v1": sets.NewString("node-1"), 169 | "gcr.io/200:v1": sets.NewString("node-1"), 170 | }, 171 | }, 172 | } 173 | 174 | for i, test := range tests { 175 | t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { 176 | imageMap := createImageExistenceMap(test.nodes) 177 | if !reflect.DeepEqual(test.expected, imageMap) { 178 | t.Errorf("expected: %#v, got: %#v", test.expected, imageMap) 179 | } 180 | }) 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/nodeports/node_ports.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package nodeports 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/kubernetes/pkg/scheduler/framework" 26 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" 27 | ) 28 | 29 | // NodePorts is a plugin that checks if a node has free ports for the requested pod ports. 30 | type NodePorts struct{} 31 | 32 | var _ framework.PreFilterPlugin = &NodePorts{} 33 | var _ framework.FilterPlugin = &NodePorts{} 34 | var _ framework.EnqueueExtensions = &NodePorts{} 35 | 36 | const ( 37 | // Name is the name of the plugin used in the plugin registry and configurations. 38 | Name = names.NodePorts 39 | 40 | // preFilterStateKey is the key in CycleState to NodePorts pre-computed data. 41 | // Using the name of the plugin will likely help us avoid collisions with other plugins. 42 | preFilterStateKey = "PreFilter" + Name 43 | 44 | // ErrReason when node ports aren't available. 45 | ErrReason = "node(s) didn't have free ports for the requested pod ports" 46 | ) 47 | 48 | type preFilterState []*v1.ContainerPort 49 | 50 | // Clone the prefilter state. 51 | func (s preFilterState) Clone() framework.StateData { 52 | // The state is not impacted by adding/removing existing pods, hence we don't need to make a deep copy. 53 | return s 54 | } 55 | 56 | // Name returns name of the plugin. It is used in logs, etc. 57 | func (pl *NodePorts) Name() string { 58 | return Name 59 | } 60 | 61 | // getContainerPorts returns the used host ports of Pods: if 'port' was used, a 'port:true' pair 62 | // will be in the result; but it does not resolve port conflict. 63 | func getContainerPorts(pods ...*v1.Pod) []*v1.ContainerPort { 64 | ports := []*v1.ContainerPort{} 65 | for _, pod := range pods { 66 | for j := range pod.Spec.Containers { 67 | container := &pod.Spec.Containers[j] 68 | for k := range container.Ports { 69 | ports = append(ports, &container.Ports[k]) 70 | } 71 | } 72 | } 73 | return ports 74 | } 75 | 76 | // PreFilter invoked at the prefilter extension point. 77 | func (pl *NodePorts) PreFilter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod) *framework.Status { 78 | s := getContainerPorts(pod) 79 | cycleState.Write(preFilterStateKey, preFilterState(s)) 80 | return nil 81 | } 82 | 83 | // PreFilterExtensions do not exist for this plugin. 84 | func (pl *NodePorts) PreFilterExtensions() framework.PreFilterExtensions { 85 | return nil 86 | } 87 | 88 | func getPreFilterState(cycleState *framework.CycleState) (preFilterState, error) { 89 | c, err := cycleState.Read(preFilterStateKey) 90 | if err != nil { 91 | // preFilterState doesn't exist, likely PreFilter wasn't invoked. 92 | return nil, fmt.Errorf("reading %q from cycleState: %w", preFilterStateKey, err) 93 | } 94 | 95 | s, ok := c.(preFilterState) 96 | if !ok { 97 | return nil, fmt.Errorf("%+v convert to nodeports.preFilterState error", c) 98 | } 99 | return s, nil 100 | } 101 | 102 | // EventsToRegister returns the possible events that may make a Pod 103 | // failed by this plugin schedulable. 104 | func (pl *NodePorts) EventsToRegister() []framework.ClusterEvent { 105 | return []framework.ClusterEvent{ 106 | // Due to immutable fields `spec.containers[*].ports`, pod update events are ignored. 107 | {Resource: framework.Pod, ActionType: framework.Delete}, 108 | {Resource: framework.Node, ActionType: framework.Add}, 109 | } 110 | } 111 | 112 | // Filter invoked at the filter extension point. 113 | func (pl *NodePorts) Filter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 114 | wantPorts, err := getPreFilterState(cycleState) 115 | if err != nil { 116 | return framework.AsStatus(err) 117 | } 118 | 119 | fits := fitsPorts(wantPorts, nodeInfo) 120 | if !fits { 121 | return framework.NewStatus(framework.Unschedulable, ErrReason) 122 | } 123 | 124 | return nil 125 | } 126 | 127 | // Fits checks if the pod fits the node. 128 | func Fits(pod *v1.Pod, nodeInfo *framework.NodeInfo) bool { 129 | return fitsPorts(getContainerPorts(pod), nodeInfo) 130 | } 131 | 132 | func fitsPorts(wantPorts []*v1.ContainerPort, nodeInfo *framework.NodeInfo) bool { 133 | // try to see whether existingPorts and wantPorts will conflict or not 134 | existingPorts := nodeInfo.UsedPorts 135 | for _, cp := range wantPorts { 136 | if existingPorts.CheckConflict(cp.HostIP, string(cp.Protocol), cp.HostPort) { 137 | return false 138 | } 139 | } 140 | return true 141 | } 142 | 143 | // New initializes a new plugin and returns it. 144 | func New(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) { 145 | return &NodePorts{}, nil 146 | } 147 | -------------------------------------------------------------------------------- /scheduler/framework/runtime/waiting_pods_map.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package runtime 18 | 19 | import ( 20 | "fmt" 21 | "sync" 22 | "time" 23 | 24 | "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/types" 26 | "k8s.io/kubernetes/pkg/scheduler/framework" 27 | ) 28 | 29 | // waitingPodsMap a thread-safe map used to maintain pods waiting in the permit phase. 30 | type waitingPodsMap struct { 31 | pods map[types.UID]*waitingPod 32 | mu sync.RWMutex 33 | } 34 | 35 | // newWaitingPodsMap returns a new waitingPodsMap. 36 | func newWaitingPodsMap() *waitingPodsMap { 37 | return &waitingPodsMap{ 38 | pods: make(map[types.UID]*waitingPod), 39 | } 40 | } 41 | 42 | // add a new WaitingPod to the map. 43 | func (m *waitingPodsMap) add(wp *waitingPod) { 44 | m.mu.Lock() 45 | defer m.mu.Unlock() 46 | m.pods[wp.GetPod().UID] = wp 47 | } 48 | 49 | // remove a WaitingPod from the map. 50 | func (m *waitingPodsMap) remove(uid types.UID) { 51 | m.mu.Lock() 52 | defer m.mu.Unlock() 53 | delete(m.pods, uid) 54 | } 55 | 56 | // get a WaitingPod from the map. 57 | func (m *waitingPodsMap) get(uid types.UID) *waitingPod { 58 | m.mu.RLock() 59 | defer m.mu.RUnlock() 60 | return m.pods[uid] 61 | } 62 | 63 | // iterate acquires a read lock and iterates over the WaitingPods map. 64 | func (m *waitingPodsMap) iterate(callback func(framework.WaitingPod)) { 65 | m.mu.RLock() 66 | defer m.mu.RUnlock() 67 | for _, v := range m.pods { 68 | callback(v) 69 | } 70 | } 71 | 72 | // waitingPod represents a pod waiting in the permit phase. 73 | type waitingPod struct { 74 | pod *v1.Pod 75 | pendingPlugins map[string]*time.Timer 76 | s chan *framework.Status 77 | mu sync.RWMutex 78 | } 79 | 80 | var _ framework.WaitingPod = &waitingPod{} 81 | 82 | // newWaitingPod returns a new waitingPod instance. 83 | func newWaitingPod(pod *v1.Pod, pluginsMaxWaitTime map[string]time.Duration) *waitingPod { 84 | wp := &waitingPod{ 85 | pod: pod, 86 | // Allow() and Reject() calls are non-blocking. This property is guaranteed 87 | // by using non-blocking send to this channel. This channel has a buffer of size 1 88 | // to ensure that non-blocking send will not be ignored - possible situation when 89 | // receiving from this channel happens after non-blocking send. 90 | s: make(chan *framework.Status, 1), 91 | } 92 | 93 | wp.pendingPlugins = make(map[string]*time.Timer, len(pluginsMaxWaitTime)) 94 | // The time.AfterFunc calls wp.Reject which iterates through pendingPlugins map. Acquire the 95 | // lock here so that time.AfterFunc can only execute after newWaitingPod finishes. 96 | wp.mu.Lock() 97 | defer wp.mu.Unlock() 98 | for k, v := range pluginsMaxWaitTime { 99 | plugin, waitTime := k, v 100 | wp.pendingPlugins[plugin] = time.AfterFunc(waitTime, func() { 101 | msg := fmt.Sprintf("rejected due to timeout after waiting %v at plugin %v", 102 | waitTime, plugin) 103 | wp.Reject(plugin, msg) 104 | }) 105 | } 106 | 107 | return wp 108 | } 109 | 110 | // GetPod returns a reference to the waiting pod. 111 | func (w *waitingPod) GetPod() *v1.Pod { 112 | return w.pod 113 | } 114 | 115 | // GetPendingPlugins returns a list of pending permit plugin's name. 116 | func (w *waitingPod) GetPendingPlugins() []string { 117 | w.mu.RLock() 118 | defer w.mu.RUnlock() 119 | plugins := make([]string, 0, len(w.pendingPlugins)) 120 | for p := range w.pendingPlugins { 121 | plugins = append(plugins, p) 122 | } 123 | 124 | return plugins 125 | } 126 | 127 | // Allow declares the waiting pod is allowed to be scheduled by plugin pluginName. 128 | // If this is the last remaining plugin to allow, then a success signal is delivered 129 | // to unblock the pod. 130 | func (w *waitingPod) Allow(pluginName string) { 131 | w.mu.Lock() 132 | defer w.mu.Unlock() 133 | if timer, exist := w.pendingPlugins[pluginName]; exist { 134 | timer.Stop() 135 | delete(w.pendingPlugins, pluginName) 136 | } 137 | 138 | // Only signal success status after all plugins have allowed 139 | if len(w.pendingPlugins) != 0 { 140 | return 141 | } 142 | 143 | // The select clause works as a non-blocking send. 144 | // If there is no receiver, it's a no-op (default case). 145 | select { 146 | case w.s <- framework.NewStatus(framework.Success, ""): 147 | default: 148 | } 149 | } 150 | 151 | // Reject declares the waiting pod unschedulable. 152 | func (w *waitingPod) Reject(pluginName, msg string) { 153 | w.mu.RLock() 154 | defer w.mu.RUnlock() 155 | for _, timer := range w.pendingPlugins { 156 | timer.Stop() 157 | } 158 | 159 | // The select clause works as a non-blocking send. 160 | // If there is no receiver, it's a no-op (default case). 161 | select { 162 | case w.s <- framework.NewStatus(framework.Unschedulable, msg).WithFailedPlugin(pluginName): 163 | default: 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /scheduler/framework/plugins/noderesources/balanced_allocation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package noderesources 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "math" 23 | 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/kubernetes/pkg/scheduler/apis/config" 27 | "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" 28 | "k8s.io/kubernetes/pkg/scheduler/framework" 29 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" 30 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" 31 | ) 32 | 33 | // BalancedAllocation is a score plugin that calculates the difference between the cpu and memory fraction 34 | // of capacity, and prioritizes the host based on how close the two metrics are to each other. 35 | type BalancedAllocation struct { 36 | handle framework.Handle 37 | resourceAllocationScorer 38 | } 39 | 40 | var _ = framework.ScorePlugin(&BalancedAllocation{}) 41 | 42 | // BalancedAllocationName is the name of the plugin used in the plugin registry and configurations. 43 | const BalancedAllocationName = names.NodeResourcesBalancedAllocation 44 | 45 | // Name returns name of the plugin. It is used in logs, etc. 46 | func (ba *BalancedAllocation) Name() string { 47 | return BalancedAllocationName 48 | } 49 | 50 | // Score invoked at the score extension point. 51 | func (ba *BalancedAllocation) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) { 52 | nodeInfo, err := ba.handle.SnapshotSharedLister().NodeInfos().Get(nodeName) 53 | if err != nil { 54 | return 0, framework.AsStatus(fmt.Errorf("getting node %q from Snapshot: %w", nodeName, err)) 55 | } 56 | 57 | // ba.score favors nodes with balanced resource usage rate. 58 | // It calculates the standard deviation for those resources and prioritizes the node based on how close the usage of those resources is to each other. 59 | // Detail: score = (1 - std) * MaxNodeScore, where std is calculated by the root square of Σ((fraction(i)-mean)^2)/len(resources) 60 | // The algorithm is partly inspired by: 61 | // "Wei Huang et al. An Energy Efficient Virtual Machine Placement Algorithm with Balanced Resource Utilization" 62 | return ba.score(pod, nodeInfo) 63 | } 64 | 65 | // ScoreExtensions of the Score plugin. 66 | func (ba *BalancedAllocation) ScoreExtensions() framework.ScoreExtensions { 67 | return nil 68 | } 69 | 70 | // NewBalancedAllocation initializes a new plugin and returns it. 71 | func NewBalancedAllocation(baArgs runtime.Object, h framework.Handle, fts feature.Features) (framework.Plugin, error) { 72 | args, ok := baArgs.(*config.NodeResourcesBalancedAllocationArgs) 73 | if !ok { 74 | return nil, fmt.Errorf("want args to be of type NodeResourcesBalancedAllocationArgs, got %T", baArgs) 75 | } 76 | 77 | if err := validation.ValidateNodeResourcesBalancedAllocationArgs(nil, args); err != nil { 78 | return nil, err 79 | } 80 | 81 | resToWeightMap := make(resourceToWeightMap) 82 | 83 | for _, resource := range args.Resources { 84 | resToWeightMap[v1.ResourceName(resource.Name)] = resource.Weight 85 | } 86 | 87 | return &BalancedAllocation{ 88 | handle: h, 89 | resourceAllocationScorer: resourceAllocationScorer{ 90 | Name: BalancedAllocationName, 91 | scorer: balancedResourceScorer, 92 | useRequested: true, 93 | resourceToWeightMap: resToWeightMap, 94 | enablePodOverhead: fts.EnablePodOverhead, 95 | }, 96 | }, nil 97 | } 98 | 99 | func balancedResourceScorer(requested, allocable resourceToValueMap) int64 { 100 | var resourceToFractions []float64 101 | var totalFraction float64 102 | for name, value := range requested { 103 | fraction := float64(value) / float64(allocable[name]) 104 | if fraction > 1 { 105 | fraction = 1 106 | } 107 | totalFraction += fraction 108 | resourceToFractions = append(resourceToFractions, fraction) 109 | } 110 | 111 | std := 0.0 112 | 113 | // For most cases, resources are limited to cpu and memory, the std could be simplified to std := (fraction1-fraction2)/2 114 | // len(fractions) > 2: calculate std based on the well-known formula - root square of Σ((fraction(i)-mean)^2)/len(fractions) 115 | // Otherwise, set the std to zero is enough. 116 | if len(resourceToFractions) == 2 { 117 | std = math.Abs((resourceToFractions[0] - resourceToFractions[1]) / 2) 118 | 119 | } else if len(resourceToFractions) > 2 { 120 | mean := totalFraction / float64(len(resourceToFractions)) 121 | var sum float64 122 | for _, fraction := range resourceToFractions { 123 | sum = sum + (fraction-mean)*(fraction-mean) 124 | } 125 | std = math.Sqrt(sum / float64(len(resourceToFractions))) 126 | } 127 | 128 | // STD (standard deviation) is always a positive value. 1-deviation lets the score to be higher for node which has least deviation and 129 | // multiplying it with `MaxNodeScore` provides the scaling factor needed. 130 | return int64((1 - std) * float64(framework.MaxNodeScore)) 131 | } 132 | --------------------------------------------------------------------------------