Merge pull request #83998 from draveness/feature/node-affinity-score-plugin

feat(scheduler): implement node affinity as score plugin

pkg/scheduler/api/compatibility/compatibility_test.go

@@ -168,7 +168,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPrioritizers: sets.NewString(
 				"EqualPriority",
-				"NodeAffinityPriority",
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
@@ -186,6 +185,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 				},
 			},
 		},
@@ -238,7 +238,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
@@ -253,6 +252,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -310,7 +310,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -326,6 +325,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -393,7 +393,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -409,6 +408,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -489,7 +489,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -505,6 +504,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -586,7 +586,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -603,6 +602,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -689,7 +689,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
@@ -706,6 +705,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -804,7 +804,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
@@ -822,6 +821,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -922,7 +922,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
@@ -940,6 +939,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -1040,7 +1040,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
@@ -1058,6 +1057,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -1162,7 +1162,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
@@ -1180,6 +1179,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodeAffinity", Weight: 2},
 					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
@@ -1216,6 +1216,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 	}
 	scoreToPriorityMap := map[string]string{
 		"ImageLocality":       "ImageLocalityPriority",
+		"NodeAffinity":        "NodeAffinityPriority",
 		"NodePreferAvoidPods": "NodePreferAvoidPodsPriority",
 		"TaintToleration":     "TaintTolerationPriority",
 	}

pkg/scheduler/framework/plugins/default_registry.go

@@ -152,6 +152,12 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
 			return
 		})
 
+	registry.RegisterPriority(priorities.NodeAffinityPriority,
+		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+			plugins.Score = appendToPluginSet(plugins.Score, nodeaffinity.Name, &args.Weight)
+			return
+		})
+
 	registry.RegisterPriority(priorities.ImageLocalityPriority,
 		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, imagelocality.Name, &args.Weight)

pkg/scheduler/framework/plugins/nodeaffinity/BUILD

@@ -7,6 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/scheduler/algorithm/predicates:go_default_library",
+        "//pkg/scheduler/algorithm/priorities:go_default_library",
         "//pkg/scheduler/framework/plugins/migration:go_default_library",
         "//pkg/scheduler/framework/v1alpha1:go_default_library",
         "//pkg/scheduler/nodeinfo:go_default_library",

pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go

@@ -18,19 +18,24 @@ package nodeaffinity
 
 import (
 	"context"
+	"fmt"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
+	"k8s.io/kubernetes/pkg/scheduler/algorithm/priorities"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration"
 	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
 	"k8s.io/kubernetes/pkg/scheduler/nodeinfo"
 )
 
 // NodeAffinity is a plugin that checks if a pod node selector matches the node label.
-type NodeAffinity struct{}
+type NodeAffinity struct {
+	handle framework.FrameworkHandle
+}
 
-var _ = framework.FilterPlugin(&NodeAffinity{})
+var _ framework.FilterPlugin = &NodeAffinity{}
+var _ framework.ScorePlugin = &NodeAffinity{}
 
 // Name is the name of the plugin used in the plugin registry and configurations.
 const Name = "NodeAffinity"
@@ -41,12 +46,36 @@ func (pl *NodeAffinity) Name() string {
 }
 
 // Filter invoked at the filter extension point.
-func (pl *NodeAffinity) Filter(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
+func (pl *NodeAffinity) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
 	_, reasons, err := predicates.PodMatchNodeSelector(pod, nil, nodeInfo)
 	return migration.PredicateResultToFrameworkStatus(reasons, err)
 }
 
-// New initializes a new plugin and returns it.
-func New(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
-	return &NodeAffinity{}, nil
+// Score invoked at the Score extension point.
+func (pl *NodeAffinity) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
+	nodeInfo, exist := pl.handle.NodeInfoSnapshot().NodeInfoMap[nodeName]
+	if !exist {
+		return 0, framework.NewStatus(framework.Error, fmt.Sprintf("node %q does not exist in NodeInfoSnapshot", nodeName))
+	}
+
+	meta := migration.PriorityMetadata(state)
+	s, err := priorities.CalculateNodeAffinityPriorityMap(pod, meta, nodeInfo)
+	return s.Score, migration.ErrorToFrameworkStatus(err)
+}
+
+// NormalizeScore invoked after scoring all nodes.
+func (pl *NodeAffinity) NormalizeScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
+	// Note that CalculateNodeAffinityPriorityReduce doesn't use priority metadata, hence passing nil here.
+	err := priorities.CalculateNodeAffinityPriorityReduce(pod, nil, pl.handle.NodeInfoSnapshot().NodeInfoMap, scores)
+	return migration.ErrorToFrameworkStatus(err)
+}
+
+// ScoreExtensions of the Score plugin.
+func (pl *NodeAffinity) ScoreExtensions() framework.ScoreExtensions {
+	return pl
+}
+
+// New initializes a new plugin and returns it.
+func New(_ *runtime.Unknown, h framework.FrameworkHandle) (framework.Plugin, error) {
+	return &NodeAffinity{handle: h}, nil
 }

pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go

@@ -705,3 +705,173 @@ func TestNodeAffinity(t *testing.T) {
 		})
 	}
 }
+
+func TestNodeAffinityPriority(t *testing.T) {
+	label1 := map[string]string{"foo": "bar"}
+	label2 := map[string]string{"key": "value"}
+	label3 := map[string]string{"az": "az1"}
+	label4 := map[string]string{"abc": "az11", "def": "az22"}
+	label5 := map[string]string{"foo": "bar", "key": "value", "az": "az1"}
+
+	affinity1 := &v1.Affinity{
+		NodeAffinity: &v1.NodeAffinity{
+			PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{{
+				Weight: 2,
+				Preference: v1.NodeSelectorTerm{
+					MatchExpressions: []v1.NodeSelectorRequirement{{
+						Key:      "foo",
+						Operator: v1.NodeSelectorOpIn,
+						Values:   []string{"bar"},
+					}},
+				},
+			}},
+		},
+	}
+
+	affinity2 := &v1.Affinity{
+		NodeAffinity: &v1.NodeAffinity{
+			PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
+				{
+					Weight: 2,
+					Preference: v1.NodeSelectorTerm{
+						MatchExpressions: []v1.NodeSelectorRequirement{
+							{
+								Key:      "foo",
+								Operator: v1.NodeSelectorOpIn,
+								Values:   []string{"bar"},
+							},
+						},
+					},
+				},
+				{
+					Weight: 4,
+					Preference: v1.NodeSelectorTerm{
+						MatchExpressions: []v1.NodeSelectorRequirement{
+							{
+								Key:      "key",
+								Operator: v1.NodeSelectorOpIn,
+								Values:   []string{"value"},
+							},
+						},
+					},
+				},
+				{
+					Weight: 5,
+					Preference: v1.NodeSelectorTerm{
+						MatchExpressions: []v1.NodeSelectorRequirement{
+							{
+								Key:      "foo",
+								Operator: v1.NodeSelectorOpIn,
+								Values:   []string{"bar"},
+							},
+							{
+								Key:      "key",
+								Operator: v1.NodeSelectorOpIn,
+								Values:   []string{"value"},
+							},
+							{
+								Key:      "az",
+								Operator: v1.NodeSelectorOpIn,
+								Values:   []string{"az1"},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		pod          *v1.Pod
+		nodes        []*v1.Node
+		expectedList framework.NodeScoreList
+		name         string
+	}{
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{},
+				},
+			},
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}},
+			},
+			expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}},
+			name:         "all machines are same priority as NodeAffinity is nil",
+		},
+		{
+			pod: &v1.Pod{
+				Spec: v1.PodSpec{
+					Affinity: affinity1,
+				},
+			},
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label4}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}},
+			},
+			expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}},
+			name:         "no machine macthes preferred scheduling requirements in NodeAffinity of pod so all machines' priority is zero",
+		},
+		{
+			pod: &v1.Pod{
+				Spec: v1.PodSpec{
+					Affinity: affinity1,
+				},
+			},
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}},
+			},
+			expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}},
+			name:         "only machine1 matches the preferred scheduling requirements of pod",
+		},
+		{
+			pod: &v1.Pod{
+				Spec: v1.PodSpec{
+					Affinity: affinity2,
+				},
+			},
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine5", Labels: label5}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}},
+			},
+			expectedList: []framework.NodeScore{{Name: "machine1", Score: 18}, {Name: "machine5", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 36}},
+			name:         "all machines matches the preferred scheduling requirements of pod but with different priorities ",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			state := framework.NewCycleState()
+
+			fh, _ := framework.NewFramework(nil, nil, nil)
+			snapshot := fh.NodeInfoSnapshot()
+			snapshot.NodeInfoMap = schedulernodeinfo.CreateNodeNameToInfoMap(nil, test.nodes)
+
+			p, _ := New(nil, fh)
+			var gotList framework.NodeScoreList
+			for _, n := range test.nodes {
+				nodeName := n.ObjectMeta.Name
+				score, status := p.(framework.ScorePlugin).Score(context.Background(), state, test.pod, nodeName)
+				if !status.IsSuccess() {
+					t.Errorf("unexpected error: %v", status)
+				}
+				gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score})
+			}
+
+			status := p.(framework.ScorePlugin).ScoreExtensions().NormalizeScore(context.Background(), state, test.pod, gotList)
+			if !status.IsSuccess() {
+				t.Errorf("unexpected error: %v", status)
+			}
+
+			if !reflect.DeepEqual(test.expectedList, gotList) {
+				t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList)
+			}
+		})
+	}
+}

test/integration/scheduler/scheduler_test.go

@@ -140,7 +140,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 				"BalancedResourceAllocation",
 				"InterPodAffinityPriority",
 				"LeastRequestedPriority",
-				"NodeAffinityPriority",
 				"SelectorSpreadPriority",
 			),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
@@ -152,6 +151,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 1},
+					{Name: "NodeAffinity", Weight: 1},
 					{Name: "NodePreferAvoidPods", Weight: 10000},
 					{Name: "TaintToleration", Weight: 1},
 				},
@@ -211,7 +211,6 @@ kind: Policy
 				"BalancedResourceAllocation",
 				"InterPodAffinityPriority",
 				"LeastRequestedPriority",
-				"NodeAffinityPriority",
 				"SelectorSpreadPriority",
 			),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
@@ -223,6 +222,7 @@ kind: Policy
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 1},
+					{Name: "NodeAffinity", Weight: 1},
 					{Name: "NodePreferAvoidPods", Weight: 10000},
 					{Name: "TaintToleration", Weight: 1},
 				},