From 00a12c787cb55fa704576e899983e62d2a3d2dc6 Mon Sep 17 00:00:00 2001 From: draveness Date: Thu, 17 Oct 2019 21:09:59 +0800 Subject: [PATCH] feat: implement node unschedulable as a filter plugin --- .../api/compatibility/compatibility_test.go | 17 +++- pkg/scheduler/framework/plugins/BUILD | 2 + .../framework/plugins/default_registry.go | 7 ++ .../framework/plugins/nodeunschedulable/BUILD | 43 +++++++++ .../nodeunschedulable/node_unschedulable.go | 54 ++++++++++++ .../node_unschedulable_test.go | 87 +++++++++++++++++++ test/integration/scheduler/scheduler_test.go | 18 ++-- 7 files changed, 217 insertions(+), 11 deletions(-) create mode 100644 pkg/scheduler/framework/plugins/nodeunschedulable/BUILD create mode 100644 pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable.go create mode 100644 pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable_test.go diff --git a/pkg/scheduler/api/compatibility/compatibility_test.go b/pkg/scheduler/api/compatibility/compatibility_test.go index cb0dcc77ac5..eb61fb8bba3 100644 --- a/pkg/scheduler/api/compatibility/compatibility_test.go +++ b/pkg/scheduler/api/compatibility/compatibility_test.go @@ -61,6 +61,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }`, wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeResources"}, {Name: "NodeName"}, {Name: "NodePorts"}, @@ -102,6 +103,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeAffinity"}, {Name: "NodeResources"}, {Name: "VolumeRestrictions"}, @@ -147,6 +149,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -203,6 +206,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -267,6 +271,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -338,6 +343,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -419,6 +425,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -513,6 +520,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -608,6 +616,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -709,6 +718,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -823,6 +833,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -938,6 +949,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -1054,6 +1066,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -1174,6 +1187,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ), wantPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeName"}, {Name: "NodePorts"}, {Name: "NodeAffinity"}, @@ -1211,9 +1225,10 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { registeredPriorities := sets.NewString(scheduler.ListRegisteredPriorityFunctions()...) seenPredicates := sets.NewString() seenPriorities := sets.NewString() - mandatoryPredicates := sets.NewString("CheckNodeUnschedulable") + mandatoryPredicates := sets.NewString() generalPredicateFilters := []string{"NodeResources", "NodeName", "NodePorts", "NodeAffinity"} filterToPredicateMap := map[string]string{ + "NodeUnschedulable": "CheckNodeUnschedulable", "TaintToleration": "PodToleratesNodeTaints", "NodeName": "HostName", "NodePorts": "PodFitsHostPorts", diff --git a/pkg/scheduler/framework/plugins/BUILD b/pkg/scheduler/framework/plugins/BUILD index e1571787c04..c3db046e95d 100644 --- a/pkg/scheduler/framework/plugins/BUILD +++ b/pkg/scheduler/framework/plugins/BUILD @@ -16,6 +16,7 @@ go_library( "//pkg/scheduler/framework/plugins/nodeports:go_default_library", "//pkg/scheduler/framework/plugins/nodepreferavoidpods:go_default_library", "//pkg/scheduler/framework/plugins/noderesources:go_default_library", + "//pkg/scheduler/framework/plugins/nodeunschedulable:go_default_library", "//pkg/scheduler/framework/plugins/nodevolumelimits:go_default_library", "//pkg/scheduler/framework/plugins/podtopologyspread:go_default_library", "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library", @@ -49,6 +50,7 @@ filegroup( "//pkg/scheduler/framework/plugins/nodeports:all-srcs", "//pkg/scheduler/framework/plugins/nodepreferavoidpods:all-srcs", "//pkg/scheduler/framework/plugins/noderesources:all-srcs", + "//pkg/scheduler/framework/plugins/nodeunschedulable:all-srcs", "//pkg/scheduler/framework/plugins/nodevolumelimits:all-srcs", "//pkg/scheduler/framework/plugins/podtopologyspread:all-srcs", "//pkg/scheduler/framework/plugins/tainttoleration:all-srcs", diff --git a/pkg/scheduler/framework/plugins/default_registry.go b/pkg/scheduler/framework/plugins/default_registry.go index a7f6fbe5703..38d818a6d65 100644 --- a/pkg/scheduler/framework/plugins/default_registry.go +++ b/pkg/scheduler/framework/plugins/default_registry.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" @@ -60,6 +61,7 @@ func NewDefaultRegistry(args *RegistryArgs) framework.Registry { nodepreferavoidpods.Name: nodepreferavoidpods.New, nodeaffinity.Name: nodeaffinity.New, podtopologyspread.Name: podtopologyspread.New, + nodeunschedulable.Name: nodeunschedulable.New, volumebinding.Name: func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { return volumebinding.NewFromVolumeBinder(args.VolumeBinder), nil }, @@ -131,6 +133,11 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry { plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil) return }) + registry.RegisterPredicate(predicates.CheckNodeUnschedulablePred, + func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + plugins.Filter = appendToPluginSet(plugins.Filter, nodeunschedulable.Name, nil) + return + }) registry.RegisterPredicate(predicates.CheckVolumeBindingPred, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, volumebinding.Name, nil) diff --git a/pkg/scheduler/framework/plugins/nodeunschedulable/BUILD b/pkg/scheduler/framework/plugins/nodeunschedulable/BUILD new file mode 100644 index 00000000000..5eba37d7e81 --- /dev/null +++ b/pkg/scheduler/framework/plugins/nodeunschedulable/BUILD @@ -0,0 +1,43 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["node_unschedulable.go"], + importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable", + visibility = ["//visibility:public"], + deps = [ + "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/framework/plugins/migration:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//pkg/scheduler/nodeinfo:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["node_unschedulable_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/api:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//pkg/scheduler/nodeinfo:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable.go b/pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable.go new file mode 100644 index 00000000000..749a6b07b28 --- /dev/null +++ b/pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable.go @@ -0,0 +1,54 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeunschedulable + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + "k8s.io/kubernetes/pkg/scheduler/nodeinfo" +) + +// NodeUnschedulable is a plugin that priorities nodes according to the node annotation +// "scheduler.alpha.kubernetes.io/preferAvoidPods". +type NodeUnschedulable struct { +} + +var _ framework.FilterPlugin = &NodeUnschedulable{} + +// Name is the name of the plugin used in the plugin registry and configurations. +const Name = "NodeUnschedulable" + +// Name returns name of the plugin. It is used in logs, etc. +func (pl *NodeUnschedulable) Name() string { + return Name +} + +// Filter invoked at the filter extension point. +func (pl *NodeUnschedulable) Filter(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { + _, reasons, err := predicates.CheckNodeUnschedulablePredicate(pod, nil, nodeInfo) + return migration.PredicateResultToFrameworkStatus(reasons, err) +} + +// New initializes a new plugin and returns it. +func New(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &NodeUnschedulable{}, nil +} diff --git a/pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable_test.go b/pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable_test.go new file mode 100644 index 00000000000..ee006534fd9 --- /dev/null +++ b/pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeunschedulable + +import ( + "context" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" +) + +func TestNodeUnschedulable(t *testing.T) { + testCases := []struct { + name string + pod *v1.Pod + node *v1.Node + wantStatus *framework.Status + }{ + { + name: "Does not schedule pod to unschedulable node (node.Spec.Unschedulable==true)", + pod: &v1.Pod{}, + node: &v1.Node{ + Spec: v1.NodeSpec{ + Unschedulable: true, + }, + }, + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, predicates.ErrNodeUnschedulable.GetReason()), + }, + { + name: "Schedule pod to normal node", + pod: &v1.Pod{}, + node: &v1.Node{ + Spec: v1.NodeSpec{ + Unschedulable: false, + }, + }, + }, + { + name: "Schedule pod with toleration to unschedulable node (node.Spec.Unschedulable==true)", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Tolerations: []v1.Toleration{ + { + Key: schedulerapi.TaintNodeUnschedulable, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + }, + node: &v1.Node{ + Spec: v1.NodeSpec{ + Unschedulable: true, + }, + }, + }, + } + + for _, test := range testCases { + nodeInfo := schedulernodeinfo.NewNodeInfo() + nodeInfo.SetNode(test.node) + + p, _ := New(nil, nil) + gotStatus := p.(framework.FilterPlugin).Filter(context.Background(), nil, test.pod, nodeInfo) + if !reflect.DeepEqual(gotStatus, test.wantStatus) { + t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) + } + } +} diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 10d37bbe32c..a8f2016dc06 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -110,7 +110,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { ] }`, expectedPredicates: sets.NewString( - "CheckNodeUnschedulable", // mandatory predicate "PredicateOne", "PredicateTwo", ), @@ -120,6 +119,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "TaintToleration"}, }, }, @@ -130,7 +130,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "apiVersion" : "v1" }`, expectedPredicates: sets.NewString( - "CheckNodeUnschedulable", // mandatory predicate "MaxAzureDiskVolumeCount", "MaxEBSVolumeCount", "MaxGCEPDVolumeCount", @@ -143,6 +142,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeResources"}, {Name: "NodeName"}, {Name: "NodePorts"}, @@ -169,12 +169,11 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "predicates" : [], "priorities" : [] }`, - expectedPredicates: sets.NewString( - "CheckNodeUnschedulable", // mandatory predicate - ), + expectedPredicates: sets.NewString(), expectedPrioritizers: sets.NewString(), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "TaintToleration"}, }, }, @@ -192,7 +191,6 @@ priorities: weight: 5 `, expectedPredicates: sets.NewString( - "CheckNodeUnschedulable", // mandatory predicate "PredicateOne", "PredicateTwo", ), @@ -202,6 +200,7 @@ priorities: ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "TaintToleration"}, }, }, @@ -211,7 +210,6 @@ priorities: kind: Policy `, expectedPredicates: sets.NewString( - "CheckNodeUnschedulable", // mandatory predicate "MaxAzureDiskVolumeCount", "MaxEBSVolumeCount", "MaxGCEPDVolumeCount", @@ -224,6 +222,7 @@ kind: Policy ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "NodeResources"}, {Name: "NodeName"}, {Name: "NodePorts"}, @@ -249,12 +248,11 @@ kind: Policy predicates: [] priorities: [] `, - expectedPredicates: sets.NewString( - "CheckNodeUnschedulable", // mandatory predicate - ), + expectedPredicates: sets.NewString(), expectedPrioritizers: sets.NewString(), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeUnschedulable"}, {Name: "TaintToleration"}, }, },