diff --git a/pkg/scheduler/api/compatibility/compatibility_test.go b/pkg/scheduler/api/compatibility/compatibility_test.go index 4ca88818dcd..d242c7d0df9 100644 --- a/pkg/scheduler/api/compatibility/compatibility_test.go +++ b/pkg/scheduler/api/compatibility/compatibility_test.go @@ -168,7 +168,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPrioritizers: sets.NewString( "EqualPriority", - "NodeAffinityPriority", "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", @@ -186,6 +185,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, }, }, }, @@ -238,7 +238,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ @@ -253,6 +252,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -310,7 +310,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), @@ -326,6 +325,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -393,7 +393,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), @@ -409,6 +408,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -489,7 +489,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), @@ -505,6 +504,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -586,7 +586,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), @@ -603,6 +602,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -689,7 +689,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", ), @@ -706,6 +705,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -804,7 +804,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", "RequestedToCapacityRatioPriority", @@ -822,6 +821,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -922,7 +922,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", "RequestedToCapacityRatioPriority", @@ -940,6 +939,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -1040,7 +1040,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", "RequestedToCapacityRatioPriority", @@ -1058,6 +1057,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -1162,7 +1162,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", "RequestedToCapacityRatioPriority", @@ -1180,6 +1179,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodeAffinity", Weight: 2}, {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, @@ -1216,6 +1216,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { } scoreToPriorityMap := map[string]string{ "ImageLocality": "ImageLocalityPriority", + "NodeAffinity": "NodeAffinityPriority", "NodePreferAvoidPods": "NodePreferAvoidPodsPriority", "TaintToleration": "TaintTolerationPriority", } diff --git a/pkg/scheduler/framework/plugins/default_registry.go b/pkg/scheduler/framework/plugins/default_registry.go index 2452793ec24..28c17d274ac 100644 --- a/pkg/scheduler/framework/plugins/default_registry.go +++ b/pkg/scheduler/framework/plugins/default_registry.go @@ -152,6 +152,12 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry { return }) + registry.RegisterPriority(priorities.NodeAffinityPriority, + func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + plugins.Score = appendToPluginSet(plugins.Score, nodeaffinity.Name, &args.Weight) + return + }) + registry.RegisterPriority(priorities.ImageLocalityPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, imagelocality.Name, &args.Weight) diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/BUILD b/pkg/scheduler/framework/plugins/nodeaffinity/BUILD index 49924ff3672..9e507c4b7fd 100644 --- a/pkg/scheduler/framework/plugins/nodeaffinity/BUILD +++ b/pkg/scheduler/framework/plugins/nodeaffinity/BUILD @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/framework/plugins/migration:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go index 164a15e335d..1bd599503ee 100644 --- a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go +++ b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go @@ -18,19 +18,24 @@ package nodeaffinity import ( "context" + "fmt" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) // NodeAffinity is a plugin that checks if a pod node selector matches the node label. -type NodeAffinity struct{} +type NodeAffinity struct { + handle framework.FrameworkHandle +} -var _ = framework.FilterPlugin(&NodeAffinity{}) +var _ framework.FilterPlugin = &NodeAffinity{} +var _ framework.ScorePlugin = &NodeAffinity{} // Name is the name of the plugin used in the plugin registry and configurations. const Name = "NodeAffinity" @@ -41,12 +46,36 @@ func (pl *NodeAffinity) Name() string { } // Filter invoked at the filter extension point. -func (pl *NodeAffinity) Filter(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { +func (pl *NodeAffinity) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { _, reasons, err := predicates.PodMatchNodeSelector(pod, nil, nodeInfo) return migration.PredicateResultToFrameworkStatus(reasons, err) } -// New initializes a new plugin and returns it. -func New(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { - return &NodeAffinity{}, nil +// Score invoked at the Score extension point. +func (pl *NodeAffinity) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) { + nodeInfo, exist := pl.handle.NodeInfoSnapshot().NodeInfoMap[nodeName] + if !exist { + return 0, framework.NewStatus(framework.Error, fmt.Sprintf("node %q does not exist in NodeInfoSnapshot", nodeName)) + } + + meta := migration.PriorityMetadata(state) + s, err := priorities.CalculateNodeAffinityPriorityMap(pod, meta, nodeInfo) + return s.Score, migration.ErrorToFrameworkStatus(err) +} + +// NormalizeScore invoked after scoring all nodes. +func (pl *NodeAffinity) NormalizeScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status { + // Note that CalculateNodeAffinityPriorityReduce doesn't use priority metadata, hence passing nil here. + err := priorities.CalculateNodeAffinityPriorityReduce(pod, nil, pl.handle.NodeInfoSnapshot().NodeInfoMap, scores) + return migration.ErrorToFrameworkStatus(err) +} + +// ScoreExtensions of the Score plugin. +func (pl *NodeAffinity) ScoreExtensions() framework.ScoreExtensions { + return pl +} + +// New initializes a new plugin and returns it. +func New(_ *runtime.Unknown, h framework.FrameworkHandle) (framework.Plugin, error) { + return &NodeAffinity{handle: h}, nil } diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go index 0499e2993bb..a588083c6f3 100644 --- a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go +++ b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go @@ -705,3 +705,173 @@ func TestNodeAffinity(t *testing.T) { }) } } + +func TestNodeAffinityPriority(t *testing.T) { + label1 := map[string]string{"foo": "bar"} + label2 := map[string]string{"key": "value"} + label3 := map[string]string{"az": "az1"} + label4 := map[string]string{"abc": "az11", "def": "az22"} + label5 := map[string]string{"foo": "bar", "key": "value", "az": "az1"} + + affinity1 := &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{{ + Weight: 2, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }}, + }, + }}, + }, + } + + affinity2 := &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 2, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + { + Weight: 4, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "key", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, + }, + }, + { + Weight: 5, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + { + Key: "key", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value"}, + }, + { + Key: "az", + Operator: v1.NodeSelectorOpIn, + Values: []string{"az1"}, + }, + }, + }, + }, + }, + }, + } + + tests := []struct { + pod *v1.Pod + nodes []*v1.Node + expectedList framework.NodeScoreList + name string + }{ + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, + name: "all machines are same priority as NodeAffinity is nil", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: affinity1, + }, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label4}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, + name: "no machine macthes preferred scheduling requirements in NodeAffinity of pod so all machines' priority is zero", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: affinity1, + }, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, + name: "only machine1 matches the preferred scheduling requirements of pod", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: affinity2, + }, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine5", Labels: label5}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 18}, {Name: "machine5", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 36}}, + name: "all machines matches the preferred scheduling requirements of pod but with different priorities ", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + state := framework.NewCycleState() + + fh, _ := framework.NewFramework(nil, nil, nil) + snapshot := fh.NodeInfoSnapshot() + snapshot.NodeInfoMap = schedulernodeinfo.CreateNodeNameToInfoMap(nil, test.nodes) + + p, _ := New(nil, fh) + var gotList framework.NodeScoreList + for _, n := range test.nodes { + nodeName := n.ObjectMeta.Name + score, status := p.(framework.ScorePlugin).Score(context.Background(), state, test.pod, nodeName) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score}) + } + + status := p.(framework.ScorePlugin).ScoreExtensions().NormalizeScore(context.Background(), state, test.pod, gotList) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + + if !reflect.DeepEqual(test.expectedList, gotList) { + t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList) + } + }) + } +} diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 0ef57226ba3..b2fca759ad7 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -140,7 +140,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "BalancedResourceAllocation", "InterPodAffinityPriority", "LeastRequestedPriority", - "NodeAffinityPriority", "SelectorSpreadPriority", ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ @@ -152,6 +151,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 1}, + {Name: "NodeAffinity", Weight: 1}, {Name: "NodePreferAvoidPods", Weight: 10000}, {Name: "TaintToleration", Weight: 1}, }, @@ -211,7 +211,6 @@ kind: Policy "BalancedResourceAllocation", "InterPodAffinityPriority", "LeastRequestedPriority", - "NodeAffinityPriority", "SelectorSpreadPriority", ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ @@ -223,6 +222,7 @@ kind: Policy }, "ScorePlugin": { {Name: "ImageLocality", Weight: 1}, + {Name: "NodeAffinity", Weight: 1}, {Name: "NodePreferAvoidPods", Weight: 10000}, {Name: "TaintToleration", Weight: 1}, },