diff --git a/cmd/kube-scheduler/app/server_test.go b/cmd/kube-scheduler/app/server_test.go index 82e7a23585b..ff6fed02d35 100644 --- a/cmd/kube-scheduler/app/server_test.go +++ b/cmd/kube-scheduler/app/server_test.go @@ -161,6 +161,7 @@ profiles: {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"}, {Name: "VolumeBinding"}, + {Name: "NodeAffinity"}, }, "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -290,6 +291,7 @@ profiles: {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"}, {Name: "VolumeBinding"}, + {Name: "NodeAffinity"}, }, "FilterPlugin": { {Name: "NodeUnschedulable"}, diff --git a/pkg/scheduler/algorithmprovider/registry.go b/pkg/scheduler/algorithmprovider/registry.go index 2b62414beaa..809f014353b 100644 --- a/pkg/scheduler/algorithmprovider/registry.go +++ b/pkg/scheduler/algorithmprovider/registry.go @@ -82,6 +82,7 @@ func getDefaultConfig() *schedulerapi.Plugins { {Name: podtopologyspread.Name}, {Name: interpodaffinity.Name}, {Name: volumebinding.Name}, + {Name: nodeaffinity.Name}, }, }, Filter: schedulerapi.PluginSet{ diff --git a/pkg/scheduler/algorithmprovider/registry_test.go b/pkg/scheduler/algorithmprovider/registry_test.go index 1e0384e5b85..38b1c41618e 100644 --- a/pkg/scheduler/algorithmprovider/registry_test.go +++ b/pkg/scheduler/algorithmprovider/registry_test.go @@ -59,6 +59,7 @@ func TestClusterAutoscalerProvider(t *testing.T) { {Name: podtopologyspread.Name}, {Name: interpodaffinity.Name}, {Name: volumebinding.Name}, + {Name: nodeaffinity.Name}, }, }, Filter: schedulerapi.PluginSet{ @@ -150,6 +151,7 @@ func TestApplyFeatureGates(t *testing.T) { {Name: podtopologyspread.Name}, {Name: interpodaffinity.Name}, {Name: volumebinding.Name}, + {Name: nodeaffinity.Name}, }, }, Filter: schedulerapi.PluginSet{ @@ -231,6 +233,7 @@ func TestApplyFeatureGates(t *testing.T) { {Name: podtopologyspread.Name}, {Name: interpodaffinity.Name}, {Name: volumebinding.Name}, + {Name: nodeaffinity.Name}, }, }, Filter: schedulerapi.PluginSet{ diff --git a/pkg/scheduler/apis/config/testing/compatibility_test.go b/pkg/scheduler/apis/config/testing/compatibility_test.go index 9e81948a467..6eefc3c204a 100644 --- a/pkg/scheduler/apis/config/testing/compatibility_test.go +++ b/pkg/scheduler/apis/config/testing/compatibility_test.go @@ -66,6 +66,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "PreFilterPlugin": { {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, + {Name: "NodeAffinity"}, }, "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -125,6 +126,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, }, @@ -180,6 +182,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, }, @@ -240,6 +243,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, }, @@ -312,6 +316,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "InterPodAffinity"}, @@ -390,6 +395,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "InterPodAffinity"}, @@ -479,6 +485,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "InterPodAffinity"}, @@ -579,6 +586,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "InterPodAffinity"}, @@ -680,6 +688,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "VolumeBinding"}, @@ -788,6 +797,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "VolumeBinding"}, @@ -908,6 +918,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "VolumeBinding"}, @@ -1030,6 +1041,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "VolumeBinding"}, @@ -1152,6 +1164,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "VolumeBinding"}, @@ -1279,6 +1292,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "QueueSortPlugin": {{Name: "PrioritySort"}}, "PreFilterPlugin": { {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, {Name: "VolumeBinding"}, @@ -1411,6 +1425,7 @@ func TestAlgorithmProviderCompatibility(t *testing.T) { {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"}, {Name: "VolumeBinding"}, + {Name: "NodeAffinity"}, }, "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -1480,6 +1495,7 @@ func TestAlgorithmProviderCompatibility(t *testing.T) { {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"}, {Name: "VolumeBinding"}, + {Name: "NodeAffinity"}, }, "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -1568,6 +1584,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"}, {Name: "VolumeBinding"}, + {Name: "NodeAffinity"}, }, "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -1689,6 +1706,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"}, {Name: "VolumeBinding"}, + {Name: "NodeAffinity"}, }, "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -1890,6 +1908,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { PreFilter: config.PluginSet{ Disabled: []config.Plugin{ {Name: "NodeResourcesFit"}, + {Name: "NodeAffinity"}, {Name: "NodePorts"}, {Name: "InterPodAffinity"}, {Name: "PodTopologySpread"}, diff --git a/pkg/scheduler/factory_test.go b/pkg/scheduler/factory_test.go index e24840eabda..1aa9f7b771a 100644 --- a/pkg/scheduler/factory_test.go +++ b/pkg/scheduler/factory_test.go @@ -155,6 +155,7 @@ func TestCreateFromConfig(t *testing.T) { Enabled: []schedulerapi.Plugin{ {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "VolumeBinding"}, {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"}, diff --git a/pkg/scheduler/framework/plugins/legacy_registry.go b/pkg/scheduler/framework/plugins/legacy_registry.go index 06e6e85c070..e5ec1c77daf 100644 --- a/pkg/scheduler/framework/plugins/legacy_registry.go +++ b/pkg/scheduler/framework/plugins/legacy_registry.go @@ -233,6 +233,7 @@ func NewLegacyRegistry() *LegacyRegistry { plugins.Filter = appendToPluginSet(plugins.Filter, nodeports.Name, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, nodeports.Name, nil) plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil) + plugins.PreFilter = appendToPluginSet(plugins.PreFilter, nodeaffinity.Name, nil) }) registry.registerPredicateConfigProducer(PodToleratesNodeTaintsPred, func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { @@ -259,6 +260,7 @@ func NewLegacyRegistry() *LegacyRegistry { registry.registerPredicateConfigProducer(MatchNodeSelectorPred, func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil) + plugins.PreFilter = appendToPluginSet(plugins.PreFilter, nodeaffinity.Name, nil) }) registry.registerPredicateConfigProducer(CheckNodeUnschedulablePred, func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/BUILD b/pkg/scheduler/framework/plugins/nodeaffinity/BUILD index d4d3169d0ab..97593a66217 100644 --- a/pkg/scheduler/framework/plugins/nodeaffinity/BUILD +++ b/pkg/scheduler/framework/plugins/nodeaffinity/BUILD @@ -11,6 +11,7 @@ go_library( "//pkg/scheduler/framework:go_default_library", "//pkg/scheduler/framework/plugins/helper:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/component-helpers/scheduling/corev1/nodeaffinity:go_default_library", ], diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go index f0b6677f93a..8b2191528a6 100644 --- a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go +++ b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go @@ -21,6 +21,7 @@ import ( "fmt" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -46,6 +47,9 @@ const ( // preScoreStateKey is the key in CycleState to NodeAffinity pre-computed data for Scoring. preScoreStateKey = "PreScore" + Name + // preFilterStateKey is the key in CycleState to NodeAffinity pre-compute data for Filtering. + preFilterStateKey = "PreFilter" + Name + // ErrReasonPod is the reason for Pod's node affinity/selector not matching. ErrReasonPod = "node(s) didn't match Pod's node affinity/selector" @@ -58,6 +62,28 @@ func (pl *NodeAffinity) Name() string { return Name } +type preFilterState struct { + requiredNodeSelector labels.Selector + requiredNodeAffinity *nodeaffinity.LazyErrorNodeSelector +} + +// Clone just returns the same state because it is not affected by pod additions or deletions. +func (s *preFilterState) Clone() framework.StateData { + return s +} + +// PreFilter builds and writes cycle state used by Filter. +func (pl *NodeAffinity) PreFilter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod) *framework.Status { + state := getPodRequiredNodeSelectorAndAffinity(pod) + cycleState.Write(preFilterStateKey, state) + return nil +} + +// PreFilterExtensions not necessary for this plugin as state doesn't depend on pod additions or deletions. +func (pl *NodeAffinity) PreFilterExtensions() framework.PreFilterExtensions { + return nil +} + // Filter checks if the Node matches the Pod .spec.affinity.nodeAffinity and // the plugin's added affinity. func (pl *NodeAffinity) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { @@ -68,8 +94,25 @@ func (pl *NodeAffinity) Filter(ctx context.Context, state *framework.CycleState, if pl.addedNodeSelector != nil && !pl.addedNodeSelector.Match(node) { return framework.NewStatus(framework.UnschedulableAndUnresolvable, errReasonEnforced) } - if !pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) { - return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod) + + s, err := getPreFilterState(state) + if err != nil { + // Fallback to calculate requiredNodeSelector and requiredNodeAffinity + // here when PreFilter is disabled. + s = getPodRequiredNodeSelectorAndAffinity(pod) + } + + if s.requiredNodeSelector != nil { + if !s.requiredNodeSelector.Matches(labels.Set(node.Labels)) { + return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod) + } + } + if s.requiredNodeAffinity != nil { + // Ignore parsing errors for backwards compatibility. + matches, _ := s.requiredNodeAffinity.Match(node) + if !matches { + return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod) + } } return nil } @@ -122,7 +165,7 @@ func (pl *NodeAffinity) Score(ctx context.Context, state *framework.CycleState, s, err := getPreScoreState(state) if err != nil { - // fallback to calculate preferredNodeAffinity here when PreScore is disabled + // Fallback to calculate preferredNodeAffinity here when PreScore is disabled. preferredNodeAffinity, err := getPodPreferredNodeAffinity(pod) if err != nil { return 0, framework.AsStatus(err) @@ -204,3 +247,31 @@ func getPreScoreState(cycleState *framework.CycleState) (*preScoreState, error) } return s, nil } + +func getPodRequiredNodeSelectorAndAffinity(pod *v1.Pod) *preFilterState { + var selector labels.Selector + if len(pod.Spec.NodeSelector) > 0 { + selector = labels.SelectorFromSet(pod.Spec.NodeSelector) + } + // Use LazyErrorNodeSelector for backwards compatibility of parsing errors. + var affinity *nodeaffinity.LazyErrorNodeSelector + if pod.Spec.Affinity != nil && + pod.Spec.Affinity.NodeAffinity != nil && + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + affinity = nodeaffinity.NewLazyErrorNodeSelector(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + } + return &preFilterState{requiredNodeSelector: selector, requiredNodeAffinity: affinity} +} + +func getPreFilterState(cycleState *framework.CycleState) (*preFilterState, error) { + c, err := cycleState.Read(preFilterStateKey) + if err != nil { + return nil, fmt.Errorf("reading %q from cycleState: %v", preFilterStateKey, err) + } + + s, ok := c.(*preFilterState) + if !ok { + return nil, fmt.Errorf("invalid PreFilter state, got type %T", c) + } + return s, nil +} diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go index 633ae805df9..dfe257a59b2 100644 --- a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go +++ b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity_test.go @@ -33,18 +33,20 @@ import ( // TODO: Add test case for RequiredDuringSchedulingRequiredDuringExecution after it's implemented. func TestNodeAffinity(t *testing.T) { tests := []struct { - pod *v1.Pod - labels map[string]string - nodeName string - name string - wantStatus *framework.Status - args config.NodeAffinityArgs + name string + pod *v1.Pod + labels map[string]string + nodeName string + wantStatus *framework.Status + args config.NodeAffinityArgs + disablePreFilter bool }{ { - pod: &v1.Pod{}, name: "no selector", + pod: &v1.Pod{}, }, { + name: "missing labels", pod: &v1.Pod{ Spec: v1.PodSpec{ NodeSelector: map[string]string{ @@ -52,10 +54,10 @@ func TestNodeAffinity(t *testing.T) { }, }, }, - name: "missing labels", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "same labels", pod: &v1.Pod{ Spec: v1.PodSpec{ NodeSelector: map[string]string{ @@ -66,9 +68,9 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "same labels", }, { + name: "node labels are superset", pod: &v1.Pod{ Spec: v1.PodSpec{ NodeSelector: map[string]string{ @@ -80,9 +82,9 @@ func TestNodeAffinity(t *testing.T) { "foo": "bar", "baz": "blah", }, - name: "node labels are superset", }, { + name: "node labels are subset", pod: &v1.Pod{ Spec: v1.PodSpec{ NodeSelector: map[string]string{ @@ -94,10 +96,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "node labels are subset", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with matchExpressions using In operator that matches the existing node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -122,9 +124,9 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with matchExpressions using In operator that matches the existing node", }, { + name: "Pod with matchExpressions using Gt operator that matches the existing node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -150,9 +152,9 @@ func TestNodeAffinity(t *testing.T) { // We use two digit to denote major version and two digit for minor version. "kernel-version": "0206", }, - name: "Pod with matchExpressions using Gt operator that matches the existing node", }, { + name: "Pod with matchExpressions using NotIn operator that matches the existing node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -177,9 +179,9 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "mem-type": "DDR3", }, - name: "Pod with matchExpressions using NotIn operator that matches the existing node", }, { + name: "Pod with matchExpressions using Exists operator that matches the existing node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -203,9 +205,9 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "GPU": "NVIDIA-GRID-K1", }, - name: "Pod with matchExpressions using Exists operator that matches the existing node", }, { + name: "Pod with affinity that don't match node's labels won't schedule onto the node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -230,10 +232,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with affinity that don't match node's labels won't schedule onto the node", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with a nil []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -248,10 +250,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with a nil []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with an empty []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -266,10 +268,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with an empty []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with empty MatchExpressions is not a valid value will match no objects and won't schedule onto the node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -288,17 +290,17 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with empty MatchExpressions is not a valid value will match no objects and won't schedule onto the node", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { - pod: &v1.Pod{}, + name: "Pod with no Affinity will schedule onto a node", + pod: &v1.Pod{}, labels: map[string]string{ "foo": "bar", }, - name: "Pod with no Affinity will schedule onto a node", }, { + name: "Pod with Affinity but nil NodeSelector will schedule onto a node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -311,9 +313,9 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with Affinity but nil NodeSelector will schedule onto a node", }, { + name: "Pod with multiple matchExpressions ANDed that matches the existing node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -341,9 +343,9 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "GPU": "NVIDIA-GRID-K1", }, - name: "Pod with multiple matchExpressions ANDed that matches the existing node", }, { + name: "Pod with multiple matchExpressions ANDed that doesn't match the existing node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -371,10 +373,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "GPU": "NVIDIA-GRID-K1", }, - name: "Pod with multiple matchExpressions ANDed that doesn't match the existing node", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with multiple NodeSelectorTerms ORed in affinity, matches the node's labels and will schedule onto the node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -408,9 +410,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with multiple NodeSelectorTerms ORed in affinity, matches the node's labels and will schedule onto the node", }, { + name: "Pod with an Affinity and a PodSpec.NodeSelector(the old thing that we are deprecating) " + + "both are satisfied, will schedule onto the node", pod: &v1.Pod{ Spec: v1.PodSpec{ NodeSelector: map[string]string{ @@ -437,10 +440,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with an Affinity and a PodSpec.NodeSelector(the old thing that we are deprecating) " + - "both are satisfied, will schedule onto the node", }, { + name: "Pod with an Affinity matches node's labels but the PodSpec.NodeSelector(the old thing that we are deprecating) " + + "is not satisfied, won't schedule onto the node", pod: &v1.Pod{ Spec: v1.PodSpec{ NodeSelector: map[string]string{ @@ -467,11 +470,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "barrrrrr", }, - name: "Pod with an Affinity matches node's labels but the PodSpec.NodeSelector(the old thing that we are deprecating) " + - "is not satisfied, won't schedule onto the node", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with an invalid value in Affinity term won't be scheduled onto the node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -496,10 +498,10 @@ func TestNodeAffinity(t *testing.T) { labels: map[string]string{ "foo": "bar", }, - name: "Pod with an invalid value in Affinity term won't be scheduled onto the node", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with matchFields using In operator that matches the existing node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -522,9 +524,9 @@ func TestNodeAffinity(t *testing.T) { }, }, nodeName: "node_1", - name: "Pod with matchFields using In operator that matches the existing node", }, { + name: "Pod with matchFields using In operator that does not match the existing node", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -547,10 +549,10 @@ func TestNodeAffinity(t *testing.T) { }, }, nodeName: "node_2", - name: "Pod with matchFields using In operator that does not match the existing node", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with two terms: matchFields does not match, but matchExpressions matches", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -583,9 +585,9 @@ func TestNodeAffinity(t *testing.T) { }, nodeName: "node_2", labels: map[string]string{"foo": "bar"}, - name: "Pod with two terms: matchFields does not match, but matchExpressions matches", }, { + name: "Pod with one term: matchFields does not match, but matchExpressions matches", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -616,10 +618,10 @@ func TestNodeAffinity(t *testing.T) { }, nodeName: "node_2", labels: map[string]string{"foo": "bar"}, - name: "Pod with one term: matchFields does not match, but matchExpressions matches", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Pod with one term: both matchFields and matchExpressions match", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -650,9 +652,9 @@ func TestNodeAffinity(t *testing.T) { }, nodeName: "node_1", labels: map[string]string{"foo": "bar"}, - name: "Pod with one term: both matchFields and matchExpressions match", }, { + name: "Pod with two terms: both matchFields and matchExpressions do not match", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -685,10 +687,10 @@ func TestNodeAffinity(t *testing.T) { }, nodeName: "node_2", labels: map[string]string{"foo": "bar"}, - name: "Pod with two terms: both matchFields and matchExpressions do not match", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Matches added affinity and Pod's node affinity", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -725,9 +727,9 @@ func TestNodeAffinity(t *testing.T) { }, }, }, - name: "Matches added affinity and Pod's node affinity", }, { + name: "Matches added affinity but not Pod's node affinity", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: &v1.Affinity{ @@ -764,10 +766,10 @@ func TestNodeAffinity(t *testing.T) { }, }, }, - name: "Matches added affinity but not Pod's node affinity", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod), }, { + name: "Doesn't match added affinity", pod: &v1.Pod{}, nodeName: "node_2", labels: map[string]string{"zone": "foo"}, @@ -786,9 +788,54 @@ func TestNodeAffinity(t *testing.T) { }, }, }, - name: "Doesn't match added affinity", wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, errReasonEnforced), }, + { + name: "Matches node selector correctly even if PreFilter is not called", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "foo": "bar", + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + "baz": "blah", + }, + disablePreFilter: true, + }, + { + name: "Matches node affinity correctly even if PreFilter is not called", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "GPU", + Operator: v1.NodeSelectorOpExists, + }, { + Key: "GPU", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"AMD", "INTER"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "GPU": "NVIDIA-GRID-K1", + }, + disablePreFilter: true, + }, } for _, test := range tests { @@ -804,7 +851,16 @@ func TestNodeAffinity(t *testing.T) { if err != nil { t.Fatalf("Creating plugin: %v", err) } - gotStatus := p.(framework.FilterPlugin).Filter(context.Background(), nil, test.pod, nodeInfo) + + state := framework.NewCycleState() + var gotStatus *framework.Status + if !test.disablePreFilter { + gotStatus = p.(framework.PreFilterPlugin).PreFilter(context.Background(), state, test.pod) + if !gotStatus.IsSuccess() { + t.Errorf("unexpected error: %v", gotStatus) + } + } + gotStatus = p.(framework.FilterPlugin).Filter(context.Background(), state, test.pod, nodeInfo) if !reflect.DeepEqual(gotStatus, test.wantStatus) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) } @@ -888,14 +944,15 @@ func TestNodeAffinityPriority(t *testing.T) { } tests := []struct { + name string pod *v1.Pod nodes []*v1.Node expectedList framework.NodeScoreList - name string args config.NodeAffinityArgs disablePreScore bool }{ { + name: "all machines are same priority as NodeAffinity is nil", pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, @@ -907,9 +964,9 @@ func TestNodeAffinityPriority(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}}, }, expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, - name: "all machines are same priority as NodeAffinity is nil", }, { + name: "no machine macthes preferred scheduling requirements in NodeAffinity of pod so all machines' priority is zero", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: affinity1, @@ -921,9 +978,9 @@ func TestNodeAffinityPriority(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}}, }, expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, - name: "no machine macthes preferred scheduling requirements in NodeAffinity of pod so all machines' priority is zero", }, { + name: "only machine1 matches the preferred scheduling requirements of pod", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: affinity1, @@ -935,9 +992,9 @@ func TestNodeAffinityPriority(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}}, }, expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, - name: "only machine1 matches the preferred scheduling requirements of pod", }, { + name: "all machines matches the preferred scheduling requirements of pod but with different priorities ", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: affinity2, @@ -949,21 +1006,21 @@ func TestNodeAffinityPriority(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}}, }, expectedList: []framework.NodeScore{{Name: "machine1", Score: 18}, {Name: "machine5", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 36}}, - name: "all machines matches the preferred scheduling requirements of pod but with different priorities ", }, { - pod: &v1.Pod{}, + name: "added affinity", + pod: &v1.Pod{}, nodes: []*v1.Node{ {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}}, {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}}, }, expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}}, - name: "added affinity", args: config.NodeAffinityArgs{ AddedAffinity: affinity1.NodeAffinity, }, }, { + name: "added affinity and pod has default affinity", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: affinity1, @@ -975,7 +1032,6 @@ func TestNodeAffinityPriority(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label5}}, }, expectedList: []framework.NodeScore{{Name: "machine1", Score: 40}, {Name: "machine2", Score: 60}, {Name: "machine3", Score: framework.MaxNodeScore}}, - name: "added affinity and pod has default affinity", args: config.NodeAffinityArgs{ AddedAffinity: &v1.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ @@ -996,6 +1052,7 @@ func TestNodeAffinityPriority(t *testing.T) { }, }, { + name: "calculate the priorities correctly even if PreScore is not called", pod: &v1.Pod{ Spec: v1.PodSpec{ Affinity: affinity2, @@ -1007,7 +1064,6 @@ func TestNodeAffinityPriority(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}}, }, expectedList: []framework.NodeScore{{Name: "machine1", Score: 18}, {Name: "machine5", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 36}}, - name: "calculate the priorities correctly even if PreScore is not called", disablePreScore: true, }, } diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 1b955d065ea..2249aac03f3 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -107,6 +107,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "PreFilterPlugin": { {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "VolumeBinding"}, {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"}, @@ -202,6 +203,7 @@ kind: Policy "PreFilterPlugin": { {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "VolumeBinding"}, {Name: "PodTopologySpread"}, {Name: "InterPodAffinity"},