diff --git a/pkg/scheduler/algorithm/BUILD b/pkg/scheduler/algorithm/BUILD index 4010474104a..3af37df6cd3 100644 --- a/pkg/scheduler/algorithm/BUILD +++ b/pkg/scheduler/algorithm/BUILD @@ -12,7 +12,6 @@ go_library( importpath = "k8s.io/kubernetes/pkg/scheduler/algorithm", deps = [ "//pkg/apis/apps:go_default_library", - "//pkg/apis/core:go_default_library", "//pkg/scheduler/apis/extender/v1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/BUILD b/pkg/scheduler/algorithm/predicates/BUILD index 8bc39153544..15df4235adf 100644 --- a/pkg/scheduler/algorithm/predicates/BUILD +++ b/pkg/scheduler/algorithm/predicates/BUILD @@ -18,11 +18,10 @@ go_library( deps = [ "//pkg/apis/core/v1/helper:go_default_library", "//pkg/features:go_default_library", - "//pkg/scheduler/algorithm:go_default_library", + "//pkg/scheduler/framework/plugins/helper:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 879c5c7c5b7..dd1dfd54961 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -22,13 +22,11 @@ import ( "k8s.io/klog" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/scheduler/algorithm" + pluginhelper "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" schedutil "k8s.io/kubernetes/pkg/scheduler/util" ) @@ -237,70 +235,13 @@ func PodFitsResourcesPredicate(pod *v1.Pod, podRequest *schedulernodeinfo.Resour return len(predicateFails) == 0, predicateFails, nil } -// nodeMatchesNodeSelectorTerms checks if a node's labels satisfy a list of node selector terms, -// terms are ORed, and an empty list of terms will match nothing. -func nodeMatchesNodeSelectorTerms(node *v1.Node, nodeSelectorTerms []v1.NodeSelectorTerm) bool { - nodeFields := map[string]string{} - for k, f := range algorithm.NodeFieldSelectorKeys { - nodeFields[k] = f(node) - } - return v1helper.MatchNodeSelectorTerms(nodeSelectorTerms, labels.Set(node.Labels), fields.Set(nodeFields)) -} - -// PodMatchesNodeSelectorAndAffinityTerms checks whether the pod is schedulable onto nodes according to -// the requirements in both NodeAffinity and nodeSelector. -func PodMatchesNodeSelectorAndAffinityTerms(pod *v1.Pod, node *v1.Node) bool { - // Check if node.Labels match pod.Spec.NodeSelector. - if len(pod.Spec.NodeSelector) > 0 { - selector := labels.SelectorFromSet(pod.Spec.NodeSelector) - if !selector.Matches(labels.Set(node.Labels)) { - return false - } - } - - // 1. nil NodeSelector matches all nodes (i.e. does not filter out any nodes) - // 2. nil []NodeSelectorTerm (equivalent to non-nil empty NodeSelector) matches no nodes - // 3. zero-length non-nil []NodeSelectorTerm matches no nodes also, just for simplicity - // 4. nil []NodeSelectorRequirement (equivalent to non-nil empty NodeSelectorTerm) matches no nodes - // 5. zero-length non-nil []NodeSelectorRequirement matches no nodes also, just for simplicity - // 6. non-nil empty NodeSelectorRequirement is not allowed - nodeAffinityMatches := true - affinity := pod.Spec.Affinity - if affinity != nil && affinity.NodeAffinity != nil { - nodeAffinity := affinity.NodeAffinity - // if no required NodeAffinity requirements, will do no-op, means select all nodes. - // TODO: Replace next line with subsequent commented-out line when implement RequiredDuringSchedulingRequiredDuringExecution. - if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { - // if nodeAffinity.RequiredDuringSchedulingRequiredDuringExecution == nil && nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { - return true - } - - // Match node selector for requiredDuringSchedulingRequiredDuringExecution. - // TODO: Uncomment this block when implement RequiredDuringSchedulingRequiredDuringExecution. - // if nodeAffinity.RequiredDuringSchedulingRequiredDuringExecution != nil { - // nodeSelectorTerms := nodeAffinity.RequiredDuringSchedulingRequiredDuringExecution.NodeSelectorTerms - // klog.V(10).Infof("Match for RequiredDuringSchedulingRequiredDuringExecution node selector terms %+v", nodeSelectorTerms) - // nodeAffinityMatches = nodeMatchesNodeSelectorTerms(node, nodeSelectorTerms) - // } - - // Match node selector for requiredDuringSchedulingIgnoredDuringExecution. - if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { - nodeSelectorTerms := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - klog.V(10).Infof("Match for RequiredDuringSchedulingIgnoredDuringExecution node selector terms %+v", nodeSelectorTerms) - nodeAffinityMatches = nodeAffinityMatches && nodeMatchesNodeSelectorTerms(node, nodeSelectorTerms) - } - - } - return nodeAffinityMatches -} - // PodMatchNodeSelector checks if a pod node selector matches the node label. func PodMatchNodeSelector(pod *v1.Pod, meta Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) { node := nodeInfo.Node() if node == nil { return false, nil, fmt.Errorf("node not found") } - if PodMatchesNodeSelectorAndAffinityTerms(pod, node) { + if pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) { return true, nil, nil } return false, []PredicateFailureReason{ErrNodeSelectorNotMatch}, nil diff --git a/pkg/scheduler/algorithm/types.go b/pkg/scheduler/algorithm/types.go index 1d8d0574404..95f39979f6f 100644 --- a/pkg/scheduler/algorithm/types.go +++ b/pkg/scheduler/algorithm/types.go @@ -23,15 +23,8 @@ import ( appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/kubernetes/pkg/apis/apps" - api "k8s.io/kubernetes/pkg/apis/core" ) -// NodeFieldSelectorKeys is a map that: the keys are node field selector keys; the values are -// the functions to get the value of the node field. -var NodeFieldSelectorKeys = map[string]func(*v1.Node) string{ - api.ObjectNameField: func(n *v1.Node) string { return n.Name }, -} - var _ corelisters.ReplicationControllerLister = &EmptyControllerLister{} // EmptyControllerLister implements ControllerLister on []v1.ReplicationController returning empty data diff --git a/pkg/scheduler/framework/plugins/helper/BUILD b/pkg/scheduler/framework/plugins/helper/BUILD index 95a3f08dcf8..020176414ce 100644 --- a/pkg/scheduler/framework/plugins/helper/BUILD +++ b/pkg/scheduler/framework/plugins/helper/BUILD @@ -9,17 +9,28 @@ go_library( importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper", visibility = ["//visibility:public"], deps = [ - "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/apis/core:go_default_library", + "//pkg/apis/core/v1/helper:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", ], ) go_test( name = "go_default_test", - srcs = ["normalize_score_test.go"], + srcs = [ + "node_affinity_test.go", + "normalize_score_test.go", + ], embed = [":go_default_library"], - deps = ["//pkg/scheduler/framework/v1alpha1:go_default_library"], + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], ) filegroup( diff --git a/pkg/scheduler/framework/plugins/helper/node_affinity.go b/pkg/scheduler/framework/plugins/helper/node_affinity.go index 4133a2ac792..f013c7c81ba 100644 --- a/pkg/scheduler/framework/plugins/helper/node_affinity.go +++ b/pkg/scheduler/framework/plugins/helper/node_affinity.go @@ -17,13 +17,63 @@ limitations under the License. package helper import ( - v1 "k8s.io/api/core/v1" - predicates "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + api "k8s.io/kubernetes/pkg/apis/core" + v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" ) // PodMatchesNodeSelectorAndAffinityTerms checks whether the pod is schedulable onto nodes according to // the requirements in both NodeAffinity and nodeSelector. func PodMatchesNodeSelectorAndAffinityTerms(pod *v1.Pod, node *v1.Node) bool { - // TODO(ahg-g): actually move the logic here. - return predicates.PodMatchesNodeSelectorAndAffinityTerms(pod, node) + // Check if node.Labels match pod.Spec.NodeSelector. + if len(pod.Spec.NodeSelector) > 0 { + selector := labels.SelectorFromSet(pod.Spec.NodeSelector) + if !selector.Matches(labels.Set(node.Labels)) { + return false + } + } + + // 1. nil NodeSelector matches all nodes (i.e. does not filter out any nodes) + // 2. nil []NodeSelectorTerm (equivalent to non-nil empty NodeSelector) matches no nodes + // 3. zero-length non-nil []NodeSelectorTerm matches no nodes also, just for simplicity + // 4. nil []NodeSelectorRequirement (equivalent to non-nil empty NodeSelectorTerm) matches no nodes + // 5. zero-length non-nil []NodeSelectorRequirement matches no nodes also, just for simplicity + // 6. non-nil empty NodeSelectorRequirement is not allowed + nodeAffinityMatches := true + affinity := pod.Spec.Affinity + if affinity != nil && affinity.NodeAffinity != nil { + nodeAffinity := affinity.NodeAffinity + // if no required NodeAffinity requirements, will do no-op, means select all nodes. + // TODO: Replace next line with subsequent commented-out line when implement RequiredDuringSchedulingRequiredDuringExecution. + if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + // if nodeAffinity.RequiredDuringSchedulingRequiredDuringExecution == nil && nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + return true + } + + // Match node selector for requiredDuringSchedulingRequiredDuringExecution. + // TODO: Uncomment this block when implement RequiredDuringSchedulingRequiredDuringExecution. + // if nodeAffinity.RequiredDuringSchedulingRequiredDuringExecution != nil { + // nodeSelectorTerms := nodeAffinity.RequiredDuringSchedulingRequiredDuringExecution.NodeSelectorTerms + // klog.V(10).Infof("Match for RequiredDuringSchedulingRequiredDuringExecution node selector terms %+v", nodeSelectorTerms) + // nodeAffinityMatches = nodeMatchesNodeSelectorTerms(node, nodeSelectorTerms) + // } + + // Match node selector for requiredDuringSchedulingIgnoredDuringExecution. + if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + nodeSelectorTerms := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + nodeAffinityMatches = nodeAffinityMatches && nodeMatchesNodeSelectorTerms(node, nodeSelectorTerms) + } + + } + return nodeAffinityMatches +} + +// nodeMatchesNodeSelectorTerms checks if a node's labels satisfy a list of node selector terms, +// terms are ORed, and an empty list of terms will match nothing. +func nodeMatchesNodeSelectorTerms(node *v1.Node, nodeSelectorTerms []v1.NodeSelectorTerm) bool { + return v1helper.MatchNodeSelectorTerms(nodeSelectorTerms, labels.Set(node.Labels), fields.Set{ + api.ObjectNameField: node.Name, + }) } diff --git a/pkg/scheduler/framework/plugins/helper/node_affinity_test.go b/pkg/scheduler/framework/plugins/helper/node_affinity_test.go new file mode 100644 index 00000000000..d11caea4258 --- /dev/null +++ b/pkg/scheduler/framework/plugins/helper/node_affinity_test.go @@ -0,0 +1,711 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + api "k8s.io/kubernetes/pkg/apis/core" + "testing" +) + +func TestPodMatchesNodeSelectorAndAffinityTerms(t *testing.T) { + tests := []struct { + name string + pod *v1.Pod + labels map[string]string + nodeName string + want bool + }{ + { + name: "no selector", + pod: &v1.Pod{}, + want: true, + }, + { + name: "missing labels", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "foo": "bar", + }, + }, + }, + want: false, + }, + { + name: "same labels", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "foo": "bar", + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: true, + }, + { + name: "node labels are superset", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "foo": "bar", + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + "baz": "blah", + }, + want: true, + }, + { + name: "node labels are subset", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "foo": "bar", + "baz": "blah", + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: false, + }, + { + name: "Pod with matchExpressions using In operator that matches the existing node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar", "value2"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: true, + }, + { + name: "Pod with matchExpressions using Gt operator that matches the existing node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "kernel-version", + Operator: v1.NodeSelectorOpGt, + Values: []string{"0204"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + // We use two digit to denote major version and two digit for minor version. + "kernel-version": "0206", + }, + want: true, + }, + { + name: "Pod with matchExpressions using NotIn operator that matches the existing node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "mem-type", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"DDR", "DDR2"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "mem-type": "DDR3", + }, + want: true, + }, + { + name: "Pod with matchExpressions using Exists operator that matches the existing node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "GPU", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "GPU": "NVIDIA-GRID-K1", + }, + want: true, + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value1", "value2"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: false, + name: "Pod with affinity that don't match node's labels won't schedule onto the node", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: nil, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: false, + name: "Pod with a nil []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{}, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: false, + name: "Pod with an empty []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node", + }, + { + name: "Pod with empty MatchExpressions is not a valid value will match no objects and won't schedule onto the node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{}, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: false, + }, + { + name: "Pod with no Affinity will schedule onto a node", + pod: &v1.Pod{}, + labels: map[string]string{ + "foo": "bar", + }, + want: true, + }, + { + name: "Pod with Affinity but nil NodeSelector will schedule onto a node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: nil, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: true, + }, + { + name: "Pod with multiple matchExpressions ANDed that matches the existing node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "GPU", + Operator: v1.NodeSelectorOpExists, + }, { + Key: "GPU", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"AMD", "INTER"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "GPU": "NVIDIA-GRID-K1", + }, + want: true, + }, + { + name: "Pod with multiple matchExpressions ANDed that doesn't match the existing node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "GPU", + Operator: v1.NodeSelectorOpExists, + }, { + Key: "GPU", + Operator: v1.NodeSelectorOpIn, + Values: []string{"AMD", "INTER"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "GPU": "NVIDIA-GRID-K1", + }, + want: false, + }, + { + name: "Pod with multiple NodeSelectorTerms ORed in affinity, matches the node's labels and will schedule onto the node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar", "value2"}, + }, + }, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "diffkey", + Operator: v1.NodeSelectorOpIn, + Values: []string{"wrong", "value2"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: true, + }, + { + name: "Pod with an Affinity and a PodSpec.NodeSelector(the old thing that we are deprecating) " + + "both are satisfied, will schedule onto the node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "foo": "bar", + }, + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: true, + }, + { + name: "Pod with an Affinity matches node's labels but the PodSpec.NodeSelector(the old thing that we are deprecating) " + + "is not satisfied, won't schedule onto the node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "foo": "bar", + }, + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "barrrrrr", + }, + want: false, + }, + { + name: "Pod with an invalid value in Affinity term won't be scheduled onto the node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"invalid value: ___@#$%^"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + labels: map[string]string{ + "foo": "bar", + }, + want: false, + }, + { + name: "Pod with matchFields using In operator that matches the existing node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: api.ObjectNameField, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_1", + want: true, + }, + { + name: "Pod with matchFields using In operator that does not match the existing node", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: api.ObjectNameField, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_2", + want: false, + }, + { + name: "Pod with two terms: matchFields does not match, but matchExpressions matches", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: api.ObjectNameField, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_2", + labels: map[string]string{"foo": "bar"}, + want: true, + }, + { + name: "Pod with one term: matchFields does not match, but matchExpressions matches", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: api.ObjectNameField, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_2", + labels: map[string]string{"foo": "bar"}, + want: false, + }, + { + name: "Pod with one term: both matchFields and matchExpressions match", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: api.ObjectNameField, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_1", + labels: map[string]string{"foo": "bar"}, + want: true, + }, + { + name: "Pod with two terms: both matchFields and matchExpressions do not match", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: api.ObjectNameField, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"not-match-to-bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_2", + labels: map[string]string{"foo": "bar"}, + want: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + node := v1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: test.nodeName, + Labels: test.labels, + }} + got := PodMatchesNodeSelectorAndAffinityTerms(test.pod, &node) + if test.want != got { + t.Errorf("expected: %v got %v", test.want, got) + } + }) + } +} diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/BUILD b/pkg/scheduler/framework/plugins/nodeaffinity/BUILD index 576baa32d71..75847691881 100644 --- a/pkg/scheduler/framework/plugins/nodeaffinity/BUILD +++ b/pkg/scheduler/framework/plugins/nodeaffinity/BUILD @@ -9,7 +9,6 @@ go_library( "//pkg/apis/core/v1/helper:go_default_library", "//pkg/scheduler/algorithm/predicates:go_default_library", "//pkg/scheduler/framework/plugins/helper:go_default_library", - "//pkg/scheduler/framework/plugins/migration:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go index cb121d4ce7c..804a1ccf878 100644 --- a/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go +++ b/pkg/scheduler/framework/plugins/nodeaffinity/node_affinity.go @@ -26,7 +26,6 @@ import ( v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" pluginhelper "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" - "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) @@ -49,8 +48,14 @@ func (pl *NodeAffinity) Name() string { // Filter invoked at the filter extension point. func (pl *NodeAffinity) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { - _, reasons, err := predicates.PodMatchNodeSelector(pod, nil, nodeInfo) - return migration.PredicateResultToFrameworkStatus(reasons, err) + node := nodeInfo.Node() + if node == nil { + return framework.NewStatus(framework.Error, "node not found") + } + if !pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) { + return framework.NewStatus(framework.UnschedulableAndUnresolvable, predicates.ErrNodeSelectorNotMatch.GetReason()) + } + return nil } // Score invoked at the Score extension point. diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/BUILD b/pkg/scheduler/framework/plugins/podtopologyspread/BUILD index 4a2dc80ca3f..f44a0d653a9 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/BUILD +++ b/pkg/scheduler/framework/plugins/podtopologyspread/BUILD @@ -12,6 +12,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/framework/plugins/helper:go_default_library", "//pkg/scheduler/framework/plugins/migration:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/listers:go_default_library", diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go index 217529d72f8..9a8792096eb 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go @@ -27,6 +27,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + pluginhelper "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/pkg/scheduler/nodeinfo" @@ -244,7 +245,7 @@ func calPreFilterState(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo) (*pr } // In accordance to design, if NodeAffinity or NodeSelector is defined, // spreading is applied to nodes that pass those filters. - if !predicates.PodMatchesNodeSelectorAndAffinityTerms(pod, node) { + if !pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) { return } diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go b/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go index e7eb8fda7d5..25b7c72e5e8 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/workqueue" "k8s.io/klog" - "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + pluginhelper "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -119,7 +119,7 @@ func (pl *PodTopologySpread) PostFilter( } // (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity // (2) All topologyKeys need to be present in `node` - if !predicates.PodMatchesNodeSelectorAndAffinityTerms(pod, node) || + if !pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) || !nodeLabelsMatchSpreadConstraints(node.Labels, state.constraints) { return }