From 2d7044a556df72af951841ee8f97c98f4a575ad5 Mon Sep 17 00:00:00 2001 From: draveness Date: Mon, 14 Oct 2019 18:28:31 +0800 Subject: [PATCH] feat(scheduler): implement NodePreferAvoidPods as score plugin --- .../api/compatibility/compatibility_test.go | 23 +-- pkg/scheduler/framework/plugins/BUILD | 2 + .../framework/plugins/default_registry.go | 20 ++- .../plugins/nodepreferavoidpods/BUILD | 41 +++++ .../node_prefer_avoid_pods.go | 65 +++++++ .../node_prefer_avoid_pods_test.go | 166 ++++++++++++++++++ test/integration/scheduler/scheduler_test.go | 4 +- 7 files changed, 302 insertions(+), 19 deletions(-) create mode 100644 pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD create mode 100644 pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go create mode 100644 pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods_test.go diff --git a/pkg/scheduler/api/compatibility/compatibility_test.go b/pkg/scheduler/api/compatibility/compatibility_test.go index cbf916c917b..4ca88818dcd 100644 --- a/pkg/scheduler/api/compatibility/compatibility_test.go +++ b/pkg/scheduler/api/compatibility/compatibility_test.go @@ -310,7 +310,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -327,6 +326,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -393,7 +393,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -410,6 +409,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -489,7 +489,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -506,6 +505,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -586,7 +586,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -604,6 +603,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -689,7 +689,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -707,6 +706,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -804,7 +804,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -823,6 +822,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -922,7 +922,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -941,6 +940,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -1040,7 +1040,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -1059,6 +1058,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -1162,7 +1162,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "LeastRequestedPriority", "BalancedResourceAllocation", "SelectorSpreadPriority", - "NodePreferAvoidPodsPriority", "NodeAffinityPriority", "InterPodAffinityPriority", "MostRequestedPriority", @@ -1181,6 +1180,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 2}, + {Name: "NodePreferAvoidPods", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, }, @@ -1215,8 +1215,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "VolumeZone": "NoVolumeZoneConflict", } scoreToPriorityMap := map[string]string{ - "TaintToleration": "TaintTolerationPriority", - "ImageLocality": "ImageLocalityPriority", + "ImageLocality": "ImageLocalityPriority", + "NodePreferAvoidPods": "NodePreferAvoidPodsPriority", + "TaintToleration": "TaintTolerationPriority", } for v, tc := range schedulerFiles { diff --git a/pkg/scheduler/framework/plugins/BUILD b/pkg/scheduler/framework/plugins/BUILD index 6c9b7196457..bf06f28e30e 100644 --- a/pkg/scheduler/framework/plugins/BUILD +++ b/pkg/scheduler/framework/plugins/BUILD @@ -14,6 +14,7 @@ go_library( "//pkg/scheduler/framework/plugins/nodeaffinity:go_default_library", "//pkg/scheduler/framework/plugins/nodename:go_default_library", "//pkg/scheduler/framework/plugins/nodeports:go_default_library", + "//pkg/scheduler/framework/plugins/nodepreferavoidpods:go_default_library", "//pkg/scheduler/framework/plugins/noderesources:go_default_library", "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library", "//pkg/scheduler/framework/plugins/volumebinding:go_default_library", @@ -45,6 +46,7 @@ filegroup( "//pkg/scheduler/framework/plugins/nodeaffinity:all-srcs", "//pkg/scheduler/framework/plugins/nodename:all-srcs", "//pkg/scheduler/framework/plugins/nodeports:all-srcs", + "//pkg/scheduler/framework/plugins/nodepreferavoidpods:all-srcs", "//pkg/scheduler/framework/plugins/noderesources:all-srcs", "//pkg/scheduler/framework/plugins/tainttoleration:all-srcs", "//pkg/scheduler/framework/plugins/volumebinding:all-srcs", diff --git a/pkg/scheduler/framework/plugins/default_registry.go b/pkg/scheduler/framework/plugins/default_registry.go index 69833e19cad..2452793ec24 100644 --- a/pkg/scheduler/framework/plugins/default_registry.go +++ b/pkg/scheduler/framework/plugins/default_registry.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding" @@ -63,12 +64,13 @@ func NewDefaultRegistry(args *RegistryArgs) framework.Registry { classInfo := &predicates.CachedStorageClassInfo{StorageClassLister: args.StorageClassLister} return framework.Registry{ - imagelocality.Name: imagelocality.New, - tainttoleration.Name: tainttoleration.New, - noderesources.Name: noderesources.New, - nodename.Name: nodename.New, - nodeports.Name: nodeports.New, - nodeaffinity.Name: nodeaffinity.New, + imagelocality.Name: imagelocality.New, + tainttoleration.Name: tainttoleration.New, + noderesources.Name: noderesources.New, + nodename.Name: nodename.New, + nodeports.Name: nodeports.New, + nodepreferavoidpods.Name: nodepreferavoidpods.New, + nodeaffinity.Name: nodeaffinity.New, volumebinding.Name: func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { return volumebinding.NewFromVolumeBinder(args.VolumeBinder), nil }, @@ -156,6 +158,12 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry { return }) + registry.RegisterPriority(priorities.NodePreferAvoidPodsPriority, + func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + plugins.Score = appendToPluginSet(plugins.Score, nodepreferavoidpods.Name, &args.Weight) + return + }) + return registry } diff --git a/pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD b/pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD new file mode 100644 index 00000000000..9db862008f9 --- /dev/null +++ b/pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD @@ -0,0 +1,41 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["node_prefer_avoid_pods.go"], + importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods", + visibility = ["//visibility:public"], + deps = [ + "//pkg/scheduler/algorithm/priorities:go_default_library", + "//pkg/scheduler/framework/plugins/migration:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["node_prefer_avoid_pods_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//pkg/scheduler/nodeinfo:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go b/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go new file mode 100644 index 00000000000..9fa8d76e6d4 --- /dev/null +++ b/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go @@ -0,0 +1,65 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodepreferavoidpods + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" +) + +// NodePreferAvoidPods is a plugin that priorities nodes according to the node annotation +// "scheduler.alpha.kubernetes.io/preferAvoidPods". +type NodePreferAvoidPods struct { + handle framework.FrameworkHandle +} + +var _ = framework.ScorePlugin(&NodePreferAvoidPods{}) + +// Name is the name of the plugin used in the plugin registry and configurations. +const Name = "NodePreferAvoidPods" + +// Name returns name of the plugin. It is used in logs, etc. +func (pl *NodePreferAvoidPods) Name() string { + return Name +} + +// Score invoked at the score extension point. +func (pl *NodePreferAvoidPods) Score(state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) { + nodeInfo, exist := pl.handle.NodeInfoSnapshot().NodeInfoMap[nodeName] + if !exist { + return 0, framework.NewStatus(framework.Error, fmt.Sprintf("node %q does not exist in NodeInfoSnapshot", nodeName)) + } + + meta := migration.PriorityMetadata(state) + s, err := priorities.CalculateNodePreferAvoidPodsPriorityMap(pod, meta, nodeInfo) + return s.Score, migration.ErrorToFrameworkStatus(err) +} + +// ScoreExtensions of the Score plugin. +func (pl *NodePreferAvoidPods) ScoreExtensions() framework.ScoreExtensions { + return nil +} + +// New initializes a new plugin and returns it. +func New(_ *runtime.Unknown, h framework.FrameworkHandle) (framework.Plugin, error) { + return &NodePreferAvoidPods{handle: h}, nil +} diff --git a/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods_test.go b/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods_test.go new file mode 100644 index 00000000000..4beaa2939a0 --- /dev/null +++ b/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods_test.go @@ -0,0 +1,166 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodepreferavoidpods + +import ( + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" +) + +func TestNodePreferAvoidPods(t *testing.T) { + annotations1 := map[string]string{ + v1.PreferAvoidPodsAnnotationKey: ` + { + "preferAvoidPods": [ + { + "podSignature": { + "podController": { + "apiVersion": "v1", + "kind": "ReplicationController", + "name": "foo", + "uid": "abcdef123456", + "controller": true + } + }, + "reason": "some reason", + "message": "some message" + } + ] + }`, + } + annotations2 := map[string]string{ + v1.PreferAvoidPodsAnnotationKey: ` + { + "preferAvoidPods": [ + { + "podSignature": { + "podController": { + "apiVersion": "v1", + "kind": "ReplicaSet", + "name": "foo", + "uid": "qwert12345", + "controller": true + } + }, + "reason": "some reason", + "message": "some message" + } + ] + }`, + } + testNodes := []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{Name: "machine1", Annotations: annotations1}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "machine2", Annotations: annotations2}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "machine3"}, + }, + } + trueVar := true + tests := []struct { + pod *v1.Pod + nodes []*v1.Node + expectedList framework.NodeScoreList + name string + }{ + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ReplicationController", Name: "foo", UID: "abcdef123456", Controller: &trueVar}, + }, + }, + }, + nodes: testNodes, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}}, + name: "pod managed by ReplicationController should avoid a node, this node get lowest priority score", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + {Kind: "RandomController", Name: "foo", UID: "abcdef123456", Controller: &trueVar}, + }, + }, + }, + nodes: testNodes, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}}, + name: "ownership by random controller should be ignored", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ReplicationController", Name: "foo", UID: "abcdef123456"}, + }, + }, + }, + nodes: testNodes, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}}, + name: "owner without Controller field set should be ignored", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ReplicaSet", Name: "foo", UID: "qwert12345", Controller: &trueVar}, + }, + }, + }, + nodes: testNodes, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: framework.MaxNodeScore}}, + name: "pod managed by ReplicaSet should avoid a node, this node get lowest priority score", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + state := framework.NewCycleState() + + fh, _ := framework.NewFramework(nil, nil, nil) + snapshot := fh.NodeInfoSnapshot() + snapshot.NodeInfoMap = schedulernodeinfo.CreateNodeNameToInfoMap(nil, test.nodes) + + p, _ := New(nil, fh) + var gotList framework.NodeScoreList + for _, n := range test.nodes { + nodeName := n.ObjectMeta.Name + score, status := p.(framework.ScorePlugin).Score(state, test.pod, nodeName) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score}) + } + + if !reflect.DeepEqual(test.expectedList, gotList) { + t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList) + } + }) + } +} diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index a0a736169fb..0ef57226ba3 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -141,7 +141,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "InterPodAffinityPriority", "LeastRequestedPriority", "NodeAffinityPriority", - "NodePreferAvoidPodsPriority", "SelectorSpreadPriority", ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ @@ -153,6 +152,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { }, "ScorePlugin": { {Name: "ImageLocality", Weight: 1}, + {Name: "NodePreferAvoidPods", Weight: 10000}, {Name: "TaintToleration", Weight: 1}, }, }, @@ -212,7 +212,6 @@ kind: Policy "InterPodAffinityPriority", "LeastRequestedPriority", "NodeAffinityPriority", - "NodePreferAvoidPodsPriority", "SelectorSpreadPriority", ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ @@ -224,6 +223,7 @@ kind: Policy }, "ScorePlugin": { {Name: "ImageLocality", Weight: 1}, + {Name: "NodePreferAvoidPods", Weight: 10000}, {Name: "TaintToleration", Weight: 1}, }, },