Merge pull request #83893 from draveness/feature/node-prefer-avoid-pods

feat(scheduler): implement NodePreferAvoidPods as score plugin

pkg/scheduler/api/compatibility/compatibility_test.go

@@ -310,7 +310,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -327,6 +326,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -393,7 +393,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -410,6 +409,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -489,7 +489,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -506,6 +505,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -586,7 +586,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -604,6 +603,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -689,7 +689,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -707,6 +706,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -804,7 +804,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -823,6 +822,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -922,7 +922,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -941,6 +940,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -1040,7 +1040,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -1059,6 +1058,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -1162,7 +1162,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"LeastRequestedPriority",
 				"BalancedResourceAllocation",
 				"SelectorSpreadPriority",
-				"NodePreferAvoidPodsPriority",
 				"NodeAffinityPriority",
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
@@ -1181,6 +1180,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 2},
+					{Name: "NodePreferAvoidPods", Weight: 2},
 					{Name: "TaintToleration", Weight: 2},
 				},
 			},
@@ -1215,8 +1215,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 		"VolumeZone":         "NoVolumeZoneConflict",
 	}
 	scoreToPriorityMap := map[string]string{
-		"TaintToleration": "TaintTolerationPriority",
+		"ImageLocality":       "ImageLocalityPriority",
-		"ImageLocality":   "ImageLocalityPriority",
+		"NodePreferAvoidPods": "NodePreferAvoidPodsPriority",
+		"TaintToleration":     "TaintTolerationPriority",
 	}
 
 	for v, tc := range schedulerFiles {

pkg/scheduler/framework/plugins/BUILD

@@ -14,6 +14,7 @@ go_library(
         "//pkg/scheduler/framework/plugins/nodeaffinity:go_default_library",
         "//pkg/scheduler/framework/plugins/nodename:go_default_library",
         "//pkg/scheduler/framework/plugins/nodeports:go_default_library",
+        "//pkg/scheduler/framework/plugins/nodepreferavoidpods:go_default_library",
         "//pkg/scheduler/framework/plugins/noderesources:go_default_library",
         "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library",
         "//pkg/scheduler/framework/plugins/volumebinding:go_default_library",
@@ -45,6 +46,7 @@ filegroup(
         "//pkg/scheduler/framework/plugins/nodeaffinity:all-srcs",
         "//pkg/scheduler/framework/plugins/nodename:all-srcs",
         "//pkg/scheduler/framework/plugins/nodeports:all-srcs",
+        "//pkg/scheduler/framework/plugins/nodepreferavoidpods:all-srcs",
         "//pkg/scheduler/framework/plugins/noderesources:all-srcs",
         "//pkg/scheduler/framework/plugins/tainttoleration:all-srcs",
         "//pkg/scheduler/framework/plugins/volumebinding:all-srcs",

pkg/scheduler/framework/plugins/default_registry.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding"
@@ -63,12 +64,13 @@ func NewDefaultRegistry(args *RegistryArgs) framework.Registry {
 	classInfo := &predicates.CachedStorageClassInfo{StorageClassLister: args.StorageClassLister}
 
 	return framework.Registry{
-		imagelocality.Name:   imagelocality.New,
+		imagelocality.Name:       imagelocality.New,
-		tainttoleration.Name: tainttoleration.New,
+		tainttoleration.Name:     tainttoleration.New,
-		noderesources.Name:   noderesources.New,
+		noderesources.Name:       noderesources.New,
-		nodename.Name:        nodename.New,
+		nodename.Name:            nodename.New,
-		nodeports.Name:       nodeports.New,
+		nodeports.Name:           nodeports.New,
-		nodeaffinity.Name:    nodeaffinity.New,
+		nodepreferavoidpods.Name: nodepreferavoidpods.New,
+		nodeaffinity.Name:        nodeaffinity.New,
 		volumebinding.Name: func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
 			return volumebinding.NewFromVolumeBinder(args.VolumeBinder), nil
 		},
@@ -156,6 +158,12 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
 			return
 		})
 
+	registry.RegisterPriority(priorities.NodePreferAvoidPodsPriority,
+		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+			plugins.Score = appendToPluginSet(plugins.Score, nodepreferavoidpods.Name, &args.Weight)
+			return
+		})
+
 	return registry
 }
 

pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD

@@ -0,0 +1,41 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["node_prefer_avoid_pods.go"],
+    importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/scheduler/algorithm/priorities:go_default_library",
+        "//pkg/scheduler/framework/plugins/migration:go_default_library",
+        "//pkg/scheduler/framework/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["node_prefer_avoid_pods_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/scheduler/framework/v1alpha1:go_default_library",
+        "//pkg/scheduler/nodeinfo:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodepreferavoidpods
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/scheduler/algorithm/priorities"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration"
+	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
+)
+
+// NodePreferAvoidPods is a plugin that priorities nodes according to the node annotation
+// "scheduler.alpha.kubernetes.io/preferAvoidPods".
+type NodePreferAvoidPods struct {
+	handle framework.FrameworkHandle
+}
+
+var _ = framework.ScorePlugin(&NodePreferAvoidPods{})
+
+// Name is the name of the plugin used in the plugin registry and configurations.
+const Name = "NodePreferAvoidPods"
+
+// Name returns name of the plugin. It is used in logs, etc.
+func (pl *NodePreferAvoidPods) Name() string {
+	return Name
+}
+
+// Score invoked at the score extension point.
+func (pl *NodePreferAvoidPods) Score(state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
+	nodeInfo, exist := pl.handle.NodeInfoSnapshot().NodeInfoMap[nodeName]
+	if !exist {
+		return 0, framework.NewStatus(framework.Error, fmt.Sprintf("node %q does not exist in NodeInfoSnapshot", nodeName))
+	}
+
+	meta := migration.PriorityMetadata(state)
+	s, err := priorities.CalculateNodePreferAvoidPodsPriorityMap(pod, meta, nodeInfo)
+	return s.Score, migration.ErrorToFrameworkStatus(err)
+}
+
+// ScoreExtensions of the Score plugin.
+func (pl *NodePreferAvoidPods) ScoreExtensions() framework.ScoreExtensions {
+	return nil
+}
+
+// New initializes a new plugin and returns it.
+func New(_ *runtime.Unknown, h framework.FrameworkHandle) (framework.Plugin, error) {
+	return &NodePreferAvoidPods{handle: h}, nil
+}

pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods_test.go

@@ -0,0 +1,166 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodepreferavoidpods
+
+import (
+	"reflect"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
+	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
+)
+
+func TestNodePreferAvoidPods(t *testing.T) {
+	annotations1 := map[string]string{
+		v1.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "apiVersion": "v1",
+							                    "kind": "ReplicationController",
+							                    "name": "foo",
+							                    "uid": "abcdef123456",
+							                    "controller": true
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+	}
+	annotations2 := map[string]string{
+		v1.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "apiVersion": "v1",
+							                    "kind": "ReplicaSet",
+							                    "name": "foo",
+							                    "uid": "qwert12345",
+							                    "controller": true
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+	}
+	testNodes := []*v1.Node{
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "machine1", Annotations: annotations1},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "machine2", Annotations: annotations2},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "machine3"},
+		},
+	}
+	trueVar := true
+	tests := []struct {
+		pod          *v1.Pod
+		nodes        []*v1.Node
+		expectedList framework.NodeScoreList
+		name         string
+	}{
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					OwnerReferences: []metav1.OwnerReference{
+						{Kind: "ReplicationController", Name: "foo", UID: "abcdef123456", Controller: &trueVar},
+					},
+				},
+			},
+			nodes:        testNodes,
+			expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}},
+			name:         "pod managed by ReplicationController should avoid a node, this node get lowest priority score",
+		},
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					OwnerReferences: []metav1.OwnerReference{
+						{Kind: "RandomController", Name: "foo", UID: "abcdef123456", Controller: &trueVar},
+					},
+				},
+			},
+			nodes:        testNodes,
+			expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}},
+			name:         "ownership by random controller should be ignored",
+		},
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					OwnerReferences: []metav1.OwnerReference{
+						{Kind: "ReplicationController", Name: "foo", UID: "abcdef123456"},
+					},
+				},
+			},
+			nodes:        testNodes,
+			expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}},
+			name:         "owner without Controller field set should be ignored",
+		},
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					OwnerReferences: []metav1.OwnerReference{
+						{Kind: "ReplicaSet", Name: "foo", UID: "qwert12345", Controller: &trueVar},
+					},
+				},
+			},
+			nodes:        testNodes,
+			expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: framework.MaxNodeScore}},
+			name:         "pod managed by ReplicaSet should avoid a node, this node get lowest priority score",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			state := framework.NewCycleState()
+
+			fh, _ := framework.NewFramework(nil, nil, nil)
+			snapshot := fh.NodeInfoSnapshot()
+			snapshot.NodeInfoMap = schedulernodeinfo.CreateNodeNameToInfoMap(nil, test.nodes)
+
+			p, _ := New(nil, fh)
+			var gotList framework.NodeScoreList
+			for _, n := range test.nodes {
+				nodeName := n.ObjectMeta.Name
+				score, status := p.(framework.ScorePlugin).Score(state, test.pod, nodeName)
+				if !status.IsSuccess() {
+					t.Errorf("unexpected error: %v", status)
+				}
+				gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score})
+			}
+
+			if !reflect.DeepEqual(test.expectedList, gotList) {
+				t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList)
+			}
+		})
+	}
+}

test/integration/scheduler/scheduler_test.go

@@ -141,7 +141,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 				"InterPodAffinityPriority",
 				"LeastRequestedPriority",
 				"NodeAffinityPriority",
-				"NodePreferAvoidPodsPriority",
 				"SelectorSpreadPriority",
 			),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
@@ -153,6 +152,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 1},
+					{Name: "NodePreferAvoidPods", Weight: 10000},
 					{Name: "TaintToleration", Weight: 1},
 				},
 			},
@@ -212,7 +212,6 @@ kind: Policy
 				"InterPodAffinityPriority",
 				"LeastRequestedPriority",
 				"NodeAffinityPriority",
-				"NodePreferAvoidPodsPriority",
 				"SelectorSpreadPriority",
 			),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
@@ -224,6 +223,7 @@ kind: Policy
 				},
 				"ScorePlugin": {
 					{Name: "ImageLocality", Weight: 1},
+					{Name: "NodePreferAvoidPods", Weight: 10000},
 					{Name: "TaintToleration", Weight: 1},
 				},
 			},