feat: implement node unschedulable as a filter plugin

This commit is contained in:
draveness 2019-10-17 21:09:59 +08:00
parent 0f7873ad5c
commit 00a12c787c
7 changed files with 217 additions and 11 deletions

View File

@ -61,6 +61,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
}`,
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeResources"},
{Name: "NodeName"},
{Name: "NodePorts"},
@ -102,6 +103,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeAffinity"},
{Name: "NodeResources"},
{Name: "VolumeRestrictions"},
@ -147,6 +149,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -203,6 +206,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -267,6 +271,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -338,6 +343,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -419,6 +425,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -513,6 +520,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -608,6 +616,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -709,6 +718,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -823,6 +833,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -938,6 +949,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -1054,6 +1066,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -1174,6 +1187,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
),
wantPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
@ -1211,9 +1225,10 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
registeredPriorities := sets.NewString(scheduler.ListRegisteredPriorityFunctions()...)
seenPredicates := sets.NewString()
seenPriorities := sets.NewString()
mandatoryPredicates := sets.NewString("CheckNodeUnschedulable")
mandatoryPredicates := sets.NewString()
generalPredicateFilters := []string{"NodeResources", "NodeName", "NodePorts", "NodeAffinity"}
filterToPredicateMap := map[string]string{
"NodeUnschedulable": "CheckNodeUnschedulable",
"TaintToleration": "PodToleratesNodeTaints",
"NodeName": "HostName",
"NodePorts": "PodFitsHostPorts",

View File

@ -16,6 +16,7 @@ go_library(
"//pkg/scheduler/framework/plugins/nodeports:go_default_library",
"//pkg/scheduler/framework/plugins/nodepreferavoidpods:go_default_library",
"//pkg/scheduler/framework/plugins/noderesources:go_default_library",
"//pkg/scheduler/framework/plugins/nodeunschedulable:go_default_library",
"//pkg/scheduler/framework/plugins/nodevolumelimits:go_default_library",
"//pkg/scheduler/framework/plugins/podtopologyspread:go_default_library",
"//pkg/scheduler/framework/plugins/tainttoleration:go_default_library",
@ -49,6 +50,7 @@ filegroup(
"//pkg/scheduler/framework/plugins/nodeports:all-srcs",
"//pkg/scheduler/framework/plugins/nodepreferavoidpods:all-srcs",
"//pkg/scheduler/framework/plugins/noderesources:all-srcs",
"//pkg/scheduler/framework/plugins/nodeunschedulable:all-srcs",
"//pkg/scheduler/framework/plugins/nodevolumelimits:all-srcs",
"//pkg/scheduler/framework/plugins/podtopologyspread:all-srcs",
"//pkg/scheduler/framework/plugins/tainttoleration:all-srcs",

View File

@ -30,6 +30,7 @@ import (
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
@ -60,6 +61,7 @@ func NewDefaultRegistry(args *RegistryArgs) framework.Registry {
nodepreferavoidpods.Name: nodepreferavoidpods.New,
nodeaffinity.Name: nodeaffinity.New,
podtopologyspread.Name: podtopologyspread.New,
nodeunschedulable.Name: nodeunschedulable.New,
volumebinding.Name: func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
return volumebinding.NewFromVolumeBinder(args.VolumeBinder), nil
},
@ -131,6 +133,11 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil)
return
})
registry.RegisterPredicate(predicates.CheckNodeUnschedulablePred,
func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
plugins.Filter = appendToPluginSet(plugins.Filter, nodeunschedulable.Name, nil)
return
})
registry.RegisterPredicate(predicates.CheckVolumeBindingPred,
func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
plugins.Filter = appendToPluginSet(plugins.Filter, volumebinding.Name, nil)

View File

@ -0,0 +1,43 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["node_unschedulable.go"],
importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable",
visibility = ["//visibility:public"],
deps = [
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/framework/plugins/migration:go_default_library",
"//pkg/scheduler/framework/v1alpha1:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["node_unschedulable_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/api:go_default_library",
"//pkg/scheduler/framework/v1alpha1:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,54 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nodeunschedulable
import (
"context"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
"k8s.io/kubernetes/pkg/scheduler/nodeinfo"
)
// NodeUnschedulable is a plugin that priorities nodes according to the node annotation
// "scheduler.alpha.kubernetes.io/preferAvoidPods".
type NodeUnschedulable struct {
}
var _ framework.FilterPlugin = &NodeUnschedulable{}
// Name is the name of the plugin used in the plugin registry and configurations.
const Name = "NodeUnschedulable"
// Name returns name of the plugin. It is used in logs, etc.
func (pl *NodeUnschedulable) Name() string {
return Name
}
// Filter invoked at the filter extension point.
func (pl *NodeUnschedulable) Filter(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
_, reasons, err := predicates.CheckNodeUnschedulablePredicate(pod, nil, nodeInfo)
return migration.PredicateResultToFrameworkStatus(reasons, err)
}
// New initializes a new plugin and returns it.
func New(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
return &NodeUnschedulable{}, nil
}

View File

@ -0,0 +1,87 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nodeunschedulable
import (
"context"
"reflect"
"testing"
v1 "k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
)
func TestNodeUnschedulable(t *testing.T) {
testCases := []struct {
name string
pod *v1.Pod
node *v1.Node
wantStatus *framework.Status
}{
{
name: "Does not schedule pod to unschedulable node (node.Spec.Unschedulable==true)",
pod: &v1.Pod{},
node: &v1.Node{
Spec: v1.NodeSpec{
Unschedulable: true,
},
},
wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, predicates.ErrNodeUnschedulable.GetReason()),
},
{
name: "Schedule pod to normal node",
pod: &v1.Pod{},
node: &v1.Node{
Spec: v1.NodeSpec{
Unschedulable: false,
},
},
},
{
name: "Schedule pod with toleration to unschedulable node (node.Spec.Unschedulable==true)",
pod: &v1.Pod{
Spec: v1.PodSpec{
Tolerations: []v1.Toleration{
{
Key: schedulerapi.TaintNodeUnschedulable,
Effect: v1.TaintEffectNoSchedule,
},
},
},
},
node: &v1.Node{
Spec: v1.NodeSpec{
Unschedulable: true,
},
},
},
}
for _, test := range testCases {
nodeInfo := schedulernodeinfo.NewNodeInfo()
nodeInfo.SetNode(test.node)
p, _ := New(nil, nil)
gotStatus := p.(framework.FilterPlugin).Filter(context.Background(), nil, test.pod, nodeInfo)
if !reflect.DeepEqual(gotStatus, test.wantStatus) {
t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus)
}
}
}

View File

@ -110,7 +110,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
]
}`,
expectedPredicates: sets.NewString(
"CheckNodeUnschedulable", // mandatory predicate
"PredicateOne",
"PredicateTwo",
),
@ -120,6 +119,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
),
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
},
},
@ -130,7 +130,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
"apiVersion" : "v1"
}`,
expectedPredicates: sets.NewString(
"CheckNodeUnschedulable", // mandatory predicate
"MaxAzureDiskVolumeCount",
"MaxEBSVolumeCount",
"MaxGCEPDVolumeCount",
@ -143,6 +142,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
),
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeResources"},
{Name: "NodeName"},
{Name: "NodePorts"},
@ -169,12 +169,11 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
"predicates" : [],
"priorities" : []
}`,
expectedPredicates: sets.NewString(
"CheckNodeUnschedulable", // mandatory predicate
),
expectedPredicates: sets.NewString(),
expectedPrioritizers: sets.NewString(),
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
},
},
@ -192,7 +191,6 @@ priorities:
weight: 5
`,
expectedPredicates: sets.NewString(
"CheckNodeUnschedulable", // mandatory predicate
"PredicateOne",
"PredicateTwo",
),
@ -202,6 +200,7 @@ priorities:
),
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
},
},
@ -211,7 +210,6 @@ priorities:
kind: Policy
`,
expectedPredicates: sets.NewString(
"CheckNodeUnschedulable", // mandatory predicate
"MaxAzureDiskVolumeCount",
"MaxEBSVolumeCount",
"MaxGCEPDVolumeCount",
@ -224,6 +222,7 @@ kind: Policy
),
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeResources"},
{Name: "NodeName"},
{Name: "NodePorts"},
@ -249,12 +248,11 @@ kind: Policy
predicates: []
priorities: []
`,
expectedPredicates: sets.NewString(
"CheckNodeUnschedulable", // mandatory predicate
),
expectedPredicates: sets.NewString(),
expectedPrioritizers: sets.NewString(),
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
},
},