From 7429b0fe865af4ce5bb1ffb81154bf8fbfa8854a Mon Sep 17 00:00:00 2001 From: Abdullah Gharaibeh Date: Sat, 5 Oct 2019 20:31:51 -0400 Subject: [PATCH] Implemented taints and tolerations priority function as a Score plugin --- pkg/scheduler/api/compatibility/BUILD | 1 + .../api/compatibility/compatibility_test.go | 111 +++--- pkg/scheduler/factory/factory_test.go | 2 +- pkg/scheduler/framework/plugins/BUILD | 1 + .../framework/plugins/default_registry.go | 7 + .../framework/plugins/migration/BUILD | 1 + .../framework/plugins/migration/utils.go | 16 + .../framework/plugins/tainttoleration/BUILD | 16 +- .../tainttoleration/taint_toleration.go | 40 ++- .../tainttoleration/taint_toleration_test.go | 337 ++++++++++++++++++ pkg/scheduler/framework/v1alpha1/framework.go | 22 +- .../framework/v1alpha1/framework_test.go | 8 +- pkg/scheduler/framework/v1alpha1/interface.go | 8 +- pkg/scheduler/internal/queue/BUILD | 1 + .../internal/queue/scheduling_queue_test.go | 3 +- test/integration/scheduler/framework_test.go | 10 +- test/integration/scheduler/scheduler_test.go | 14 +- 17 files changed, 510 insertions(+), 88 deletions(-) create mode 100644 pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go diff --git a/pkg/scheduler/api/compatibility/BUILD b/pkg/scheduler/api/compatibility/BUILD index 3ea02c5ce7b..721e863b781 100644 --- a/pkg/scheduler/api/compatibility/BUILD +++ b/pkg/scheduler/api/compatibility/BUILD @@ -16,6 +16,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//vendor/github.com/google/go-cmp/cmp:go_default_library", ], ) diff --git a/pkg/scheduler/api/compatibility/compatibility_test.go b/pkg/scheduler/api/compatibility/compatibility_test.go index 7812774c5ea..8c73c223fbe 100644 --- a/pkg/scheduler/api/compatibility/compatibility_test.go +++ b/pkg/scheduler/api/compatibility/compatibility_test.go @@ -19,6 +19,8 @@ package compatibility import ( "testing" + "github.com/google/go-cmp/cmp" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -37,11 +39,11 @@ import ( func TestCompatibility_v1_Scheduler(t *testing.T) { // Add serialized versions of scheduler config that exercise available options to ensure compatibility between releases schedulerFiles := map[string]struct { - JSON string - wantPredicates sets.String - wantPrioritizers sets.String - wantFilterPlugins sets.String - wantExtenders []schedulerapi.ExtenderConfig + JSON string + wantPredicates sets.String + wantPrioritizers sets.String + wantPlugins map[string][]kubeschedulerconfig.Plugin + wantExtenders []schedulerapi.ExtenderConfig }{ // Do not change this JSON after the corresponding release has been tagged. // A failure indicates backwards compatibility with the specified release was broken. @@ -231,12 +233,12 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "BalancedResourceAllocation", "SelectorSpreadPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, }, // Do not change this JSON after the corresponding release has been tagged. @@ -300,13 +302,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, }, // Do not change this JSON after the corresponding release has been tagged. // A failure indicates backwards compatibility with the specified release was broken. @@ -379,13 +381,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, wantExtenders: []schedulerapi.ExtenderConfig{{ URLPrefix: "/prefix", FilterVerb: "filter", @@ -471,13 +473,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, wantExtenders: []schedulerapi.ExtenderConfig{{ URLPrefix: "/prefix", FilterVerb: "filter", @@ -565,13 +567,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, wantExtenders: []schedulerapi.ExtenderConfig{{ URLPrefix: "/prefix", FilterVerb: "filter", @@ -664,13 +666,13 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, wantExtenders: []schedulerapi.ExtenderConfig{{ URLPrefix: "/prefix", FilterVerb: "filter", @@ -775,14 +777,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", "RequestedToCapacityRatioPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, wantExtenders: []schedulerapi.ExtenderConfig{{ URLPrefix: "/prefix", FilterVerb: "filter", @@ -889,14 +891,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", "RequestedToCapacityRatioPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, wantExtenders: []schedulerapi.ExtenderConfig{{ URLPrefix: "/prefix", FilterVerb: "filter", @@ -1003,14 +1005,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", "RequestedToCapacityRatioPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, wantExtenders: []schedulerapi.ExtenderConfig{{ URLPrefix: "/prefix", FilterVerb: "filter", @@ -1121,14 +1123,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "SelectorSpreadPriority", "NodePreferAvoidPodsPriority", "NodeAffinityPriority", - "TaintTolerationPriority", "InterPodAffinityPriority", "MostRequestedPriority", "RequestedToCapacityRatioPriority", ), - wantFilterPlugins: sets.NewString( - "TaintToleration", - ), + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 2}}, + }, wantExtenders: []schedulerapi.ExtenderConfig{{ URLPrefix: "/prefix", FilterVerb: "filter", @@ -1152,6 +1154,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { filterToPredicateMap := map[string]string{ "TaintToleration": "PodToleratesNodeTaints", } + scoreToPriorityMap := map[string]string{ + "TaintToleration": "TaintTolerationPriority", + } for v, tc := range schedulerFiles { t.Run(v, func(t *testing.T) { @@ -1208,15 +1213,17 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { t.Errorf("Got prioritizers %v, want %v", gotPrioritizers, tc.wantPrioritizers) } - gotFilterPlugins := sets.NewString() - plugins := sched.Framework.ListPlugins() - for _, p := range plugins["FilterPlugin"] { - gotFilterPlugins.Insert(p) - seenPredicates.Insert(filterToPredicateMap[p]) + gotPlugins := sched.Framework.ListPlugins() + for _, p := range gotPlugins["FilterPlugin"] { + seenPredicates.Insert(filterToPredicateMap[p.Name]) } - if !gotFilterPlugins.Equal(tc.wantFilterPlugins) { - t.Errorf("Got filter plugins %v, want %v", gotFilterPlugins, tc.wantFilterPlugins) + for _, p := range gotPlugins["FilterPlugin"] { + seenPriorities.Insert(scoreToPriorityMap[p.Name]) + + } + if diff := cmp.Diff(tc.wantPlugins, gotPlugins); diff != "" { + t.Errorf("unexpected plugins diff (-want, +got): %s", diff) } gotExtenders := sched.Algorithm.Extenders() diff --git a/pkg/scheduler/factory/factory_test.go b/pkg/scheduler/factory/factory_test.go index 5433a493111..12f95452d88 100644 --- a/pkg/scheduler/factory/factory_test.go +++ b/pkg/scheduler/factory/factory_test.go @@ -635,7 +635,7 @@ func (t *TestPlugin) Name() string { return t.name } -func (t *TestPlugin) Score(state *framework.CycleState, p *v1.Pod, nodeName string) (int, *framework.Status) { +func (t *TestPlugin) Score(state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) { return 1, nil } diff --git a/pkg/scheduler/framework/plugins/BUILD b/pkg/scheduler/framework/plugins/BUILD index f6bbc4ff9f9..c4ed397d6a2 100644 --- a/pkg/scheduler/framework/plugins/BUILD +++ b/pkg/scheduler/framework/plugins/BUILD @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", diff --git a/pkg/scheduler/framework/plugins/default_registry.go b/pkg/scheduler/framework/plugins/default_registry.go index 2503da18058..0beb2d9ced5 100644 --- a/pkg/scheduler/framework/plugins/default_registry.go +++ b/pkg/scheduler/framework/plugins/default_registry.go @@ -20,6 +20,7 @@ import ( "fmt" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" @@ -64,6 +65,12 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry { return }) + registry.RegisterPriority(priorities.TaintTolerationPriority, + func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + plugins.Score = appendToPluginSet(plugins.Score, tainttoleration.Name, &args.Weight) + return + }) + return registry } diff --git a/pkg/scheduler/framework/plugins/migration/BUILD b/pkg/scheduler/framework/plugins/migration/BUILD index db78246abb4..1a291e45aac 100644 --- a/pkg/scheduler/framework/plugins/migration/BUILD +++ b/pkg/scheduler/framework/plugins/migration/BUILD @@ -8,6 +8,7 @@ go_library( deps = [ "//pkg/scheduler/algorithm/predicates:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/scheduler/framework/plugins/migration/utils.go b/pkg/scheduler/framework/plugins/migration/utils.go index fb6ba859a98..3aa3870af7d 100644 --- a/pkg/scheduler/framework/plugins/migration/utils.go +++ b/pkg/scheduler/framework/plugins/migration/utils.go @@ -17,6 +17,7 @@ limitations under the License. package migration import ( + "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -88,3 +89,18 @@ func (p *PrioritiesStateData) Clone() framework.StateData { Reference: p.Reference, } } + +// PriorityMetadata returns priority metadata stored in CycleState. +func PriorityMetadata(state *framework.CycleState) interface{} { + if state == nil { + return nil + } + + var meta interface{} + if s, err := state.Read(PrioritiesStateKey); err == nil { + meta = s.(*PrioritiesStateData).Reference + } else { + klog.Errorf("reading key %q from CycleState, continuing without metadata: %v", PrioritiesStateKey, err) + } + return meta +} diff --git a/pkg/scheduler/framework/plugins/tainttoleration/BUILD b/pkg/scheduler/framework/plugins/tainttoleration/BUILD index b8b575b7e90..5831519d99d 100644 --- a/pkg/scheduler/framework/plugins/tainttoleration/BUILD +++ b/pkg/scheduler/framework/plugins/tainttoleration/BUILD @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/framework/plugins/migration:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", @@ -28,3 +29,16 @@ filegroup( tags = ["automanaged"], visibility = ["//visibility:public"], ) + +go_test( + name = "go_default_test", + srcs = ["taint_toleration_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//pkg/scheduler/nodeinfo:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go index 36490c70a45..9ed8cb5f230 100644 --- a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go +++ b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go @@ -17,18 +17,24 @@ limitations under the License. package tainttoleration import ( + "fmt" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) // TaintToleration is a plugin that checks if a pod tolerates a node's taints. -type TaintToleration struct{} +type TaintToleration struct { + handle framework.FrameworkHandle +} var _ = framework.FilterPlugin(&TaintToleration{}) +var _ = framework.ScorePlugin(&TaintToleration{}) // Name is the name of the plugin used in the plugin registry and configurations. const Name = "TaintToleration" @@ -39,12 +45,36 @@ func (pl *TaintToleration) Name() string { } // Filter invoked at the filter extension point. -func (pl *TaintToleration) Filter(_ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { +func (pl *TaintToleration) Filter(state *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { + // Note that PodToleratesNodeTaints doesn't use predicate metadata, hence passing nil here. _, reasons, err := predicates.PodToleratesNodeTaints(pod, nil, nodeInfo) return migration.PredicateResultToFrameworkStatus(reasons, err) } -// New initializes a new plugin and returns it. -func New(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { - return &TaintToleration{}, nil +// Score invoked at the Score extension point. +func (pl *TaintToleration) Score(state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) { + nodeInfo, exist := pl.handle.NodeInfoSnapshot().NodeInfoMap[nodeName] + if !exist { + return 0, framework.NewStatus(framework.Error, fmt.Sprintf("node %q does not exist in NodeInfoSnapshot", nodeName)) + } + meta := migration.PriorityMetadata(state) + s, err := priorities.ComputeTaintTolerationPriorityMap(pod, meta, nodeInfo) + return s.Score, migration.ErrorToFrameworkStatus(err) +} + +// NormalizeScore invoked after scoring all nodes. +func (pl *TaintToleration) NormalizeScore(_ *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status { + // Note that ComputeTaintTolerationPriorityReduce doesn't use priority metadata, hence passing nil here. + err := priorities.ComputeTaintTolerationPriorityReduce(pod, nil, pl.handle.NodeInfoSnapshot().NodeInfoMap, scores) + return migration.ErrorToFrameworkStatus(err) +} + +// ScoreExtensions of the Score plugin. +func (pl *TaintToleration) ScoreExtensions() framework.ScoreExtensions { + return pl +} + +// New initializes a new plugin and returns it. +func New(_ *runtime.Unknown, h framework.FrameworkHandle) (framework.Plugin, error) { + return &TaintToleration{handle: h}, nil } diff --git a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go new file mode 100644 index 00000000000..eba768ce0e6 --- /dev/null +++ b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go @@ -0,0 +1,337 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tainttoleration + +import ( + "reflect" + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" +) + +func nodeWithTaints(nodeName string, taints []v1.Taint) *v1.Node { + return &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Spec: v1.NodeSpec{ + Taints: taints, + }, + } +} + +func podWithTolerations(podName string, tolerations []v1.Toleration) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + Tolerations: tolerations, + }, + } +} + +func TestTaintTolerationScore(t *testing.T) { + tests := []struct { + name string + pod *v1.Pod + nodes []*v1.Node + expectedList framework.NodeScoreList + }{ + // basic test case + { + name: "node with taints tolerated by the pod, gets a higher score than those node with intolerable taints", + pod: podWithTolerations("pod1", []v1.Toleration{{ + Key: "foo", + Operator: v1.TolerationOpEqual, + Value: "bar", + Effect: v1.TaintEffectPreferNoSchedule, + }}), + nodes: []*v1.Node{ + nodeWithTaints("nodeA", []v1.Taint{{ + Key: "foo", + Value: "bar", + Effect: v1.TaintEffectPreferNoSchedule, + }}), + nodeWithTaints("nodeB", []v1.Taint{{ + Key: "foo", + Value: "blah", + Effect: v1.TaintEffectPreferNoSchedule, + }}), + }, + expectedList: []framework.NodeScore{ + {Name: "nodeA", Score: framework.MaxNodeScore}, + {Name: "nodeB", Score: 0}, + }, + }, + // the count of taints that are tolerated by pod, does not matter. + { + name: "the nodes that all of their taints are tolerated by the pod, get the same score, no matter how many tolerable taints a node has", + pod: podWithTolerations("pod1", []v1.Toleration{ + { + Key: "cpu-type", + Operator: v1.TolerationOpEqual, + Value: "arm64", + Effect: v1.TaintEffectPreferNoSchedule, + }, { + Key: "disk-type", + Operator: v1.TolerationOpEqual, + Value: "ssd", + Effect: v1.TaintEffectPreferNoSchedule, + }, + }), + nodes: []*v1.Node{ + nodeWithTaints("nodeA", []v1.Taint{}), + nodeWithTaints("nodeB", []v1.Taint{ + { + Key: "cpu-type", + Value: "arm64", + Effect: v1.TaintEffectPreferNoSchedule, + }, + }), + nodeWithTaints("nodeC", []v1.Taint{ + { + Key: "cpu-type", + Value: "arm64", + Effect: v1.TaintEffectPreferNoSchedule, + }, { + Key: "disk-type", + Value: "ssd", + Effect: v1.TaintEffectPreferNoSchedule, + }, + }), + }, + expectedList: []framework.NodeScore{ + {Name: "nodeA", Score: framework.MaxNodeScore}, + {Name: "nodeB", Score: framework.MaxNodeScore}, + {Name: "nodeC", Score: framework.MaxNodeScore}, + }, + }, + // the count of taints on a node that are not tolerated by pod, matters. + { + name: "the more intolerable taints a node has, the lower score it gets.", + pod: podWithTolerations("pod1", []v1.Toleration{{ + Key: "foo", + Operator: v1.TolerationOpEqual, + Value: "bar", + Effect: v1.TaintEffectPreferNoSchedule, + }}), + nodes: []*v1.Node{ + nodeWithTaints("nodeA", []v1.Taint{}), + nodeWithTaints("nodeB", []v1.Taint{ + { + Key: "cpu-type", + Value: "arm64", + Effect: v1.TaintEffectPreferNoSchedule, + }, + }), + nodeWithTaints("nodeC", []v1.Taint{ + { + Key: "cpu-type", + Value: "arm64", + Effect: v1.TaintEffectPreferNoSchedule, + }, { + Key: "disk-type", + Value: "ssd", + Effect: v1.TaintEffectPreferNoSchedule, + }, + }), + }, + expectedList: []framework.NodeScore{ + {Name: "nodeA", Score: framework.MaxNodeScore}, + {Name: "nodeB", Score: 5}, + {Name: "nodeC", Score: 0}, + }, + }, + // taints-tolerations priority only takes care about the taints and tolerations that have effect PreferNoSchedule + { + name: "only taints and tolerations that have effect PreferNoSchedule are checked by taints-tolerations priority function", + pod: podWithTolerations("pod1", []v1.Toleration{ + { + Key: "cpu-type", + Operator: v1.TolerationOpEqual, + Value: "arm64", + Effect: v1.TaintEffectNoSchedule, + }, { + Key: "disk-type", + Operator: v1.TolerationOpEqual, + Value: "ssd", + Effect: v1.TaintEffectNoSchedule, + }, + }), + nodes: []*v1.Node{ + nodeWithTaints("nodeA", []v1.Taint{}), + nodeWithTaints("nodeB", []v1.Taint{ + { + Key: "cpu-type", + Value: "arm64", + Effect: v1.TaintEffectNoSchedule, + }, + }), + nodeWithTaints("nodeC", []v1.Taint{ + { + Key: "cpu-type", + Value: "arm64", + Effect: v1.TaintEffectPreferNoSchedule, + }, { + Key: "disk-type", + Value: "ssd", + Effect: v1.TaintEffectPreferNoSchedule, + }, + }), + }, + expectedList: []framework.NodeScore{ + {Name: "nodeA", Score: framework.MaxNodeScore}, + {Name: "nodeB", Score: framework.MaxNodeScore}, + {Name: "nodeC", Score: 0}, + }, + }, + { + name: "Default behaviour No taints and tolerations, lands on node with no taints", + //pod without tolerations + pod: podWithTolerations("pod1", []v1.Toleration{}), + nodes: []*v1.Node{ + //Node without taints + nodeWithTaints("nodeA", []v1.Taint{}), + nodeWithTaints("nodeB", []v1.Taint{ + { + Key: "cpu-type", + Value: "arm64", + Effect: v1.TaintEffectPreferNoSchedule, + }, + }), + }, + expectedList: []framework.NodeScore{ + {Name: "nodeA", Score: framework.MaxNodeScore}, + {Name: "nodeB", Score: 0}, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + state := framework.NewCycleState() + + fh, _ := framework.NewFramework(nil, nil, nil) + snapshot := fh.NodeInfoSnapshot() + snapshot.NodeInfoMap = schedulernodeinfo.CreateNodeNameToInfoMap(nil, test.nodes) + + p, _ := New(nil, fh) + var gotList framework.NodeScoreList + for _, n := range test.nodes { + nodeName := n.ObjectMeta.Name + score, status := p.(framework.ScorePlugin).Score(state, test.pod, nodeName) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score}) + } + + status := p.(framework.ScorePlugin).ScoreExtensions().NormalizeScore(state, test.pod, gotList) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + + if !reflect.DeepEqual(test.expectedList, gotList) { + t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList) + } + }) + } +} + +func TestTaintTolerationFilter(t *testing.T) { + unschedulable := framework.NewStatus(framework.UnschedulableAndUnresolvable, predicates.ErrTaintsTolerationsNotMatch.GetReason()) + tests := []struct { + name string + pod *v1.Pod + node *v1.Node + wantStatus *framework.Status + }{ + { + name: "A pod having no tolerations can't be scheduled onto a node with nonempty taints", + pod: podWithTolerations("pod1", []v1.Toleration{}), + node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}), + wantStatus: unschedulable, + }, + { + name: "A pod which can be scheduled on a dedicated node assigned to user1 with effect NoSchedule", + pod: podWithTolerations("pod1", []v1.Toleration{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}), + node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}), + }, + { + name: "A pod which can't be scheduled on a dedicated node assigned to user2 with effect NoSchedule", + pod: podWithTolerations("pod1", []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}}), + node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}), + wantStatus: unschedulable, + }, + { + name: "A pod can be scheduled onto the node, with a toleration uses operator Exists that tolerates the taints on the node", + pod: podWithTolerations("pod1", []v1.Toleration{{Key: "foo", Operator: "Exists", Effect: "NoSchedule"}}), + node: nodeWithTaints("nodeA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}), + }, + { + name: "A pod has multiple tolerations, node has multiple taints, all the taints are tolerated, pod can be scheduled onto the node", + pod: podWithTolerations("pod1", []v1.Toleration{ + {Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}, + {Key: "foo", Operator: "Exists", Effect: "NoSchedule"}, + }), + node: nodeWithTaints("nodeA", []v1.Taint{ + {Key: "dedicated", Value: "user2", Effect: "NoSchedule"}, + {Key: "foo", Value: "bar", Effect: "NoSchedule"}, + }), + }, + { + name: "A pod has a toleration that keys and values match the taint on the node, but (non-empty) effect doesn't match, " + + "can't be scheduled onto the node", + pod: podWithTolerations("pod1", []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar", Effect: "PreferNoSchedule"}}), + node: nodeWithTaints("nodeA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}), + wantStatus: unschedulable, + }, + { + name: "The pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " + + "and the effect of taint is NoSchedule. Pod can be scheduled onto the node", + pod: podWithTolerations("pod1", []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar"}}), + node: nodeWithTaints("nodeA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}), + }, + { + name: "The pod has a toleration that key and value don't match the taint on the node, " + + "but the effect of taint on node is PreferNochedule. Pod can be scheduled onto the node", + pod: podWithTolerations("pod1", []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}}), + node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"}}), + }, + { + name: "The pod has no toleration, " + + "but the effect of taint on node is PreferNochedule. Pod can be scheduled onto the node", + pod: podWithTolerations("pod1", []v1.Toleration{}), + node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"}}), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + nodeInfo := schedulernodeinfo.NewNodeInfo() + nodeInfo.SetNode(test.node) + p, _ := New(nil, nil) + gotStatus := p.(framework.FilterPlugin).Filter(nil, test.pod, nodeInfo) + if !reflect.DeepEqual(gotStatus, test.wantStatus) { + t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) + } + }) + } +} diff --git a/pkg/scheduler/framework/v1alpha1/framework.go b/pkg/scheduler/framework/v1alpha1/framework.go index a0c5a3195a0..a8b5852bbf9 100644 --- a/pkg/scheduler/framework/v1alpha1/framework.go +++ b/pkg/scheduler/framework/v1alpha1/framework.go @@ -553,18 +553,24 @@ func (f *framework) GetWaitingPod(uid types.UID) WaitingPod { // ListPlugins returns a map of extension point name to plugin names configured at each extension // point. Returns nil if no plugins where configred. -func (f *framework) ListPlugins() map[string][]string { - m := make(map[string][]string) +func (f *framework) ListPlugins() map[string][]config.Plugin { + m := make(map[string][]config.Plugin) + for _, e := range f.getExtensionPoints(&config.Plugins{}) { plugins := reflect.ValueOf(e.slicePtr).Elem() - var names []string + extName := plugins.Type().Elem().Name() + var cfgs []config.Plugin for i := 0; i < plugins.Len(); i++ { name := plugins.Index(i).Interface().(Plugin).Name() - names = append(names, name) + p := config.Plugin{Name: name} + if extName == "ScorePlugin" { + // Weights apply only to score plugins. + p.Weight = int32(f.pluginNameToWeightMap[name]) + } + cfgs = append(cfgs, p) } - if len(names) > 0 { - extName := plugins.Type().Elem().Name() - m[extName] = names + if len(cfgs) > 0 { + m[extName] = cfgs } } if len(m) > 0 { @@ -579,7 +585,7 @@ func (f *framework) ClientSet() clientset.Interface { } func (f *framework) pluginsNeeded(plugins *config.Plugins) map[string]config.Plugin { - pgMap := make(map[string]config.Plugin, 0) + pgMap := make(map[string]config.Plugin) if plugins == nil { return pgMap diff --git a/pkg/scheduler/framework/v1alpha1/framework_test.go b/pkg/scheduler/framework/v1alpha1/framework_test.go index 044518a48be..b4715504207 100644 --- a/pkg/scheduler/framework/v1alpha1/framework_test.go +++ b/pkg/scheduler/framework/v1alpha1/framework_test.go @@ -85,7 +85,7 @@ func (pl *TestScoreWithNormalizePlugin) NormalizeScore(state *CycleState, pod *v return injectNormalizeRes(pl.inj, scores) } -func (pl *TestScoreWithNormalizePlugin) Score(state *CycleState, p *v1.Pod, nodeName string) (int, *Status) { +func (pl *TestScoreWithNormalizePlugin) Score(state *CycleState, p *v1.Pod, nodeName string) (int64, *Status) { return setScoreRes(pl.inj) } @@ -103,7 +103,7 @@ func (pl *TestScorePlugin) Name() string { return pl.name } -func (pl *TestScorePlugin) Score(state *CycleState, p *v1.Pod, nodeName string) (int, *Status) { +func (pl *TestScorePlugin) Score(state *CycleState, p *v1.Pod, nodeName string) (int64, *Status) { return setScoreRes(pl.inj) } @@ -523,13 +523,13 @@ func buildConfigWithWeights(weights map[string]int32, ps ...string) *config.Plug } type injectedResult struct { - ScoreRes int `json:"scoreRes,omitempty"` + ScoreRes int64 `json:"scoreRes,omitempty"` NormalizeRes int64 `json:"normalizeRes,omitempty"` ScoreErr bool `json:"scoreErr,omitempty"` NormalizeErr bool `json:"normalizeErr,omitempty"` } -func setScoreRes(inj injectedResult) (int, *Status) { +func setScoreRes(inj injectedResult) (int64, *Status) { if inj.ScoreErr { return 0, NewStatus(Error, "injecting failure.") } diff --git a/pkg/scheduler/framework/v1alpha1/interface.go b/pkg/scheduler/framework/v1alpha1/interface.go index e4e97310bfe..70c2982482c 100644 --- a/pkg/scheduler/framework/v1alpha1/interface.go +++ b/pkg/scheduler/framework/v1alpha1/interface.go @@ -26,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/scheduler/apis/config" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) @@ -252,7 +253,7 @@ type ScorePlugin interface { // Score is called on each filtered node. It must return success and an integer // indicating the rank of the node. All scoring plugins must return success or // the pod will be rejected. - Score(state *CycleState, p *v1.Pod, nodeName string) (int, *Status) + Score(state *CycleState, p *v1.Pod, nodeName string) (int64, *Status) // ScoreExtensions returns a ScoreExtensions interface if it implements one, or nil if does not. ScoreExtensions() ScoreExtensions @@ -407,9 +408,8 @@ type Framework interface { // code=4("skip") status. RunBindPlugins(state *CycleState, pod *v1.Pod, nodeName string) *Status - // ListPlugins returns a map of extension point name to plugin names - // configured at each extension point. - ListPlugins() map[string][]string + // ListPlugins returns a map of extension point name to list of configured Plugins. + ListPlugins() map[string][]config.Plugin } // FrameworkHandle provides data and some tools that plugins can use. It is diff --git a/pkg/scheduler/internal/queue/BUILD b/pkg/scheduler/internal/queue/BUILD index b79edec3fc3..fe4c4511906 100644 --- a/pkg/scheduler/internal/queue/BUILD +++ b/pkg/scheduler/internal/queue/BUILD @@ -34,6 +34,7 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/api/v1/pod:go_default_library", + "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/metrics:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go index 1f2c8159b81..5a887dd4d5b 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue_test.go +++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go @@ -31,6 +31,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/component-base/metrics/testutil" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/scheduler/apis/config" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/pkg/scheduler/metrics" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" @@ -165,7 +166,7 @@ func (*fakeFramework) QueueSortFunc() framework.LessFunc { } } -func (f *fakeFramework) ListPlugins() map[string][]string { +func (f *fakeFramework) ListPlugins() map[string][]config.Plugin { return nil } diff --git a/test/integration/scheduler/framework_test.go b/test/integration/scheduler/framework_test.go index eaf50721907..05e55a0df57 100644 --- a/test/integration/scheduler/framework_test.go +++ b/test/integration/scheduler/framework_test.go @@ -148,17 +148,17 @@ func (sp *ScorePlugin) reset() { } // Score returns the score of scheduling a pod on a specific node. -func (sp *ScorePlugin) Score(state *framework.CycleState, p *v1.Pod, nodeName string) (int, *framework.Status) { +func (sp *ScorePlugin) Score(state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) { sp.numScoreCalled++ if sp.failScore { return 0, framework.NewStatus(framework.Error, fmt.Sprintf("injecting failure for pod %v", p.Name)) } - score := 1 + score := int64(1) if sp.numScoreCalled == 1 { // The first node is scored the highest, the rest is scored lower. sp.highScoreNode = nodeName - score = int(framework.MaxNodeScore) + score = framework.MaxNodeScore } return score, nil } @@ -179,9 +179,9 @@ func (sp *ScoreWithNormalizePlugin) reset() { } // Score returns the score of scheduling a pod on a specific node. -func (sp *ScoreWithNormalizePlugin) Score(state *framework.CycleState, p *v1.Pod, nodeName string) (int, *framework.Status) { +func (sp *ScoreWithNormalizePlugin) Score(state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) { sp.numScoreCalled++ - score := 10 + score := int64(10) return score, nil } diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index d21eef44545..20db7d90157 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -95,7 +95,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { policy string expectedPredicates sets.String expectedPrioritizers sets.String - expectedPlugins map[string][]string + expectedPlugins map[string][]kubeschedulerconfig.Plugin }{ { policy: `{ @@ -147,11 +147,11 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "NodeAffinityPriority", "NodePreferAvoidPodsPriority", "SelectorSpreadPriority", - "TaintTolerationPriority", "ImageLocalityPriority", ), - expectedPlugins: map[string][]string{ - "FilterPlugin": {"TaintToleration"}, + expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 1}}, }, }, { @@ -214,11 +214,11 @@ kind: Policy "NodeAffinityPriority", "NodePreferAvoidPodsPriority", "SelectorSpreadPriority", - "TaintTolerationPriority", "ImageLocalityPriority", ), - expectedPlugins: map[string][]string{ - "FilterPlugin": {"TaintToleration"}, + expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": {{Name: "TaintToleration"}}, + "ScorePlugin": {{Name: "TaintToleration", Weight: 1}}, }, }, {