Merge pull request #84047 from draveness/feature/node-unschedulable-filter

feat: implement node unschedulable as a filter plugin

pkg/scheduler/api/compatibility/compatibility_test.go

@@ -61,6 +61,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 		}`,
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeResources"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
@@ -102,6 +103,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeAffinity"},
 					{Name: "NodeResources"},
 					{Name: "VolumeRestrictions"},
@@ -147,6 +149,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -203,6 +206,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -267,6 +271,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -338,6 +343,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -419,6 +425,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -513,6 +520,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -608,6 +616,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -709,6 +718,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -823,6 +833,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -938,6 +949,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -1054,6 +1066,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -1174,6 +1187,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 			),
 			wantPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
 					{Name: "NodeAffinity"},
@@ -1211,9 +1225,10 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 	registeredPriorities := sets.NewString(scheduler.ListRegisteredPriorityFunctions()...)
 	seenPredicates := sets.NewString()
 	seenPriorities := sets.NewString()
-	mandatoryPredicates := sets.NewString("CheckNodeUnschedulable")
+	mandatoryPredicates := sets.NewString()
 	generalPredicateFilters := []string{"NodeResources", "NodeName", "NodePorts", "NodeAffinity"}
 	filterToPredicateMap := map[string]string{
+		"NodeUnschedulable":  "CheckNodeUnschedulable",
 		"TaintToleration":    "PodToleratesNodeTaints",
 		"NodeName":           "HostName",
 		"NodePorts":          "PodFitsHostPorts",

pkg/scheduler/framework/plugins/BUILD

@@ -16,6 +16,7 @@ go_library(
         "//pkg/scheduler/framework/plugins/nodeports:go_default_library",
         "//pkg/scheduler/framework/plugins/nodepreferavoidpods:go_default_library",
         "//pkg/scheduler/framework/plugins/noderesources:go_default_library",
+        "//pkg/scheduler/framework/plugins/nodeunschedulable:go_default_library",
         "//pkg/scheduler/framework/plugins/nodevolumelimits:go_default_library",
         "//pkg/scheduler/framework/plugins/podtopologyspread:go_default_library",
         "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library",
@@ -49,6 +50,7 @@ filegroup(
         "//pkg/scheduler/framework/plugins/nodeports:all-srcs",
         "//pkg/scheduler/framework/plugins/nodepreferavoidpods:all-srcs",
         "//pkg/scheduler/framework/plugins/noderesources:all-srcs",
+        "//pkg/scheduler/framework/plugins/nodeunschedulable:all-srcs",
         "//pkg/scheduler/framework/plugins/nodevolumelimits:all-srcs",
         "//pkg/scheduler/framework/plugins/podtopologyspread:all-srcs",
         "//pkg/scheduler/framework/plugins/tainttoleration:all-srcs",

pkg/scheduler/framework/plugins/default_registry.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
@@ -60,6 +61,7 @@ func NewDefaultRegistry(args *RegistryArgs) framework.Registry {
 		nodepreferavoidpods.Name: nodepreferavoidpods.New,
 		nodeaffinity.Name:        nodeaffinity.New,
 		podtopologyspread.Name:   podtopologyspread.New,
+		nodeunschedulable.Name:   nodeunschedulable.New,
 		volumebinding.Name: func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
 			return volumebinding.NewFromVolumeBinder(args.VolumeBinder), nil
 		},
@@ -131,6 +133,11 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil)
 			return
 		})
+	registry.RegisterPredicate(predicates.CheckNodeUnschedulablePred,
+		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+			plugins.Filter = appendToPluginSet(plugins.Filter, nodeunschedulable.Name, nil)
+			return
+		})
 	registry.RegisterPredicate(predicates.CheckVolumeBindingPred,
 		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, volumebinding.Name, nil)

pkg/scheduler/framework/plugins/nodeunschedulable/BUILD

@@ -0,0 +1,43 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["node_unschedulable.go"],
+    importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/scheduler/algorithm/predicates:go_default_library",
+        "//pkg/scheduler/framework/plugins/migration:go_default_library",
+        "//pkg/scheduler/framework/v1alpha1:go_default_library",
+        "//pkg/scheduler/nodeinfo:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["node_unschedulable_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/scheduler/algorithm/predicates:go_default_library",
+        "//pkg/scheduler/api:go_default_library",
+        "//pkg/scheduler/framework/v1alpha1:go_default_library",
+        "//pkg/scheduler/nodeinfo:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeunschedulable
+
+import (
+	"context"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration"
+	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
+	"k8s.io/kubernetes/pkg/scheduler/nodeinfo"
+)
+
+// NodeUnschedulable is a plugin that priorities nodes according to the node annotation
+// "scheduler.alpha.kubernetes.io/preferAvoidPods".
+type NodeUnschedulable struct {
+}
+
+var _ framework.FilterPlugin = &NodeUnschedulable{}
+
+// Name is the name of the plugin used in the plugin registry and configurations.
+const Name = "NodeUnschedulable"
+
+// Name returns name of the plugin. It is used in logs, etc.
+func (pl *NodeUnschedulable) Name() string {
+	return Name
+}
+
+// Filter invoked at the filter extension point.
+func (pl *NodeUnschedulable) Filter(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
+	_, reasons, err := predicates.CheckNodeUnschedulablePredicate(pod, nil, nodeInfo)
+	return migration.PredicateResultToFrameworkStatus(reasons, err)
+}
+
+// New initializes a new plugin and returns it.
+func New(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
+	return &NodeUnschedulable{}, nil
+}

pkg/scheduler/framework/plugins/nodeunschedulable/node_unschedulable_test.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeunschedulable
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
+	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
+	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
+	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
+)
+
+func TestNodeUnschedulable(t *testing.T) {
+	testCases := []struct {
+		name       string
+		pod        *v1.Pod
+		node       *v1.Node
+		wantStatus *framework.Status
+	}{
+		{
+			name: "Does not schedule pod to unschedulable node (node.Spec.Unschedulable==true)",
+			pod:  &v1.Pod{},
+			node: &v1.Node{
+				Spec: v1.NodeSpec{
+					Unschedulable: true,
+				},
+			},
+			wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, predicates.ErrNodeUnschedulable.GetReason()),
+		},
+		{
+			name: "Schedule pod to normal node",
+			pod:  &v1.Pod{},
+			node: &v1.Node{
+				Spec: v1.NodeSpec{
+					Unschedulable: false,
+				},
+			},
+		},
+		{
+			name: "Schedule pod with toleration to unschedulable node (node.Spec.Unschedulable==true)",
+			pod: &v1.Pod{
+				Spec: v1.PodSpec{
+					Tolerations: []v1.Toleration{
+						{
+							Key:    schedulerapi.TaintNodeUnschedulable,
+							Effect: v1.TaintEffectNoSchedule,
+						},
+					},
+				},
+			},
+			node: &v1.Node{
+				Spec: v1.NodeSpec{
+					Unschedulable: true,
+				},
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		nodeInfo := schedulernodeinfo.NewNodeInfo()
+		nodeInfo.SetNode(test.node)
+
+		p, _ := New(nil, nil)
+		gotStatus := p.(framework.FilterPlugin).Filter(context.Background(), nil, test.pod, nodeInfo)
+		if !reflect.DeepEqual(gotStatus, test.wantStatus) {
+			t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus)
+		}
+	}
+}

test/integration/scheduler/scheduler_test.go

@@ -110,7 +110,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 				]
 			}`,
 			expectedPredicates: sets.NewString(
-				"CheckNodeUnschedulable", // mandatory predicate
 				"PredicateOne",
 				"PredicateTwo",
 			),
@@ -120,6 +119,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 			),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "TaintToleration"},
 				},
 			},
@@ -130,7 +130,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 				"apiVersion" : "v1"
 			}`,
 			expectedPredicates: sets.NewString(
-				"CheckNodeUnschedulable", // mandatory predicate
 				"MaxAzureDiskVolumeCount",
 				"MaxEBSVolumeCount",
 				"MaxGCEPDVolumeCount",
@@ -143,6 +142,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 			),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeResources"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
@@ -169,12 +169,11 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 				"predicates" : [],
 				"priorities" : []
 			}`,
-			expectedPredicates: sets.NewString(
-				"CheckNodeUnschedulable", // mandatory predicate
-			),
+			expectedPredicates:   sets.NewString(),
 			expectedPrioritizers: sets.NewString(),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "TaintToleration"},
 				},
 			},
@@ -192,7 +191,6 @@ priorities:
   weight: 5
 `,
 			expectedPredicates: sets.NewString(
-				"CheckNodeUnschedulable", // mandatory predicate
 				"PredicateOne",
 				"PredicateTwo",
 			),
@@ -202,6 +200,7 @@ priorities:
 			),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "TaintToleration"},
 				},
 			},
@@ -211,7 +210,6 @@ priorities:
 kind: Policy
 `,
 			expectedPredicates: sets.NewString(
-				"CheckNodeUnschedulable", // mandatory predicate
 				"MaxAzureDiskVolumeCount",
 				"MaxEBSVolumeCount",
 				"MaxGCEPDVolumeCount",
@@ -224,6 +222,7 @@ kind: Policy
 			),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "NodeResources"},
 					{Name: "NodeName"},
 					{Name: "NodePorts"},
@@ -249,12 +248,11 @@ kind: Policy
 predicates: []
 priorities: []
 `,
-			expectedPredicates: sets.NewString(
-				"CheckNodeUnschedulable", // mandatory predicate
-			),
+			expectedPredicates:   sets.NewString(),
 			expectedPrioritizers: sets.NewString(),
 			expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
 				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
 					{Name: "TaintToleration"},
 				},
 			},