diff --git a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go index 6b28fc33ce3..9f9463858af 100644 --- a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go +++ b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go @@ -644,8 +644,8 @@ func TestDryRunPreemption(t *testing.T) { nodeNames: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"}, testPods: []*v1.Pod{ st.MakePod().Name("p").UID("p").Label("foo", "").Priority(highPriority). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), }, initPods: []*v1.Pod{ @@ -1486,8 +1486,8 @@ func TestPreempt(t *testing.T) { { name: "preemption for topology spread constraints", pod: st.MakePod().Name("p").UID("p").Namespace(v1.NamespaceDefault).Label("foo", "").Priority(highPriority). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), pods: []*v1.Pod{ st.MakePod().Name("p-a1").UID("p-a1").Namespace(v1.NamespaceDefault).Node("node-a").Label("foo", "").Priority(highPriority).Obj(), diff --git a/pkg/scheduler/framework/plugins/feature/feature.go b/pkg/scheduler/framework/plugins/feature/feature.go index 4f24a1c4c97..ed8284a4306 100644 --- a/pkg/scheduler/framework/plugins/feature/feature.go +++ b/pkg/scheduler/framework/plugins/feature/feature.go @@ -20,9 +20,10 @@ package feature // This struct allows us to break the dependency of the plugins on // the internal k8s features pkg. type Features struct { - EnablePodAffinityNamespaceSelector bool - EnablePodDisruptionBudget bool - EnableReadWriteOncePod bool - EnableVolumeCapacityPriority bool - EnableMinDomainsInPodTopologySpread bool + EnablePodAffinityNamespaceSelector bool + EnablePodDisruptionBudget bool + EnableReadWriteOncePod bool + EnableVolumeCapacityPriority bool + EnableMinDomainsInPodTopologySpread bool + EnableNodeInclusionPolicyInPodTopologySpread bool } diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/common.go b/pkg/scheduler/framework/plugins/podtopologyspread/common.go index a6cefc89dcc..29c61a4c770 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/common.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/common.go @@ -20,6 +20,8 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + v1helper "k8s.io/component-helpers/scheduling/corev1" + "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" ) @@ -33,17 +35,35 @@ type topologyPair struct { // and where the selector is parsed. // Fields are exported for comparison during testing. type topologySpreadConstraint struct { - MaxSkew int32 - TopologyKey string - Selector labels.Selector - MinDomains int32 + MaxSkew int32 + TopologyKey string + Selector labels.Selector + MinDomains int32 + NodeAffinityPolicy v1.NodeInclusionPolicy + NodeTaintsPolicy v1.NodeInclusionPolicy +} + +func (tsc *topologySpreadConstraint) matchNodeInclusionPolicies(pod *v1.Pod, node *v1.Node, require nodeaffinity.RequiredNodeAffinity) bool { + if tsc.NodeAffinityPolicy == v1.NodeInclusionPolicyHonor { + // We ignore parsing errors here for backwards compatibility. + if match, _ := require.Match(node); !match { + return false + } + } + + if tsc.NodeTaintsPolicy == v1.NodeInclusionPolicyHonor { + if _, untolerated := v1helper.FindMatchingUntoleratedTaint(node.Spec.Taints, pod.Spec.Tolerations, nil); untolerated { + return false + } + } + return true } // buildDefaultConstraints builds the constraints for a pod using // .DefaultConstraints and the selectors from the services, replication // controllers, replica sets and stateful sets that match the pod. func (pl *PodTopologySpread) buildDefaultConstraints(p *v1.Pod, action v1.UnsatisfiableConstraintAction) ([]topologySpreadConstraint, error) { - constraints, err := filterTopologySpreadConstraints(pl.defaultConstraints, action, pl.enableMinDomainsInPodTopologySpread) + constraints, err := filterTopologySpreadConstraints(pl.defaultConstraints, action, pl.enableMinDomainsInPodTopologySpread, pl.enableNodeInclusionPolicyInPodTopologySpread) if err != nil || len(constraints) == 0 { return nil, err } @@ -67,7 +87,7 @@ func nodeLabelsMatchSpreadConstraints(nodeLabels map[string]string, constraints return true } -func filterTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint, action v1.UnsatisfiableConstraintAction, enableMinDomainsInPodTopologySpread bool) ([]topologySpreadConstraint, error) { +func filterTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint, action v1.UnsatisfiableConstraintAction, enableMinDomainsInPodTopologySpread, enableNodeInclusionPolicyInPodTopologySpread bool) ([]topologySpreadConstraint, error) { var result []topologySpreadConstraint for _, c := range constraints { if c.WhenUnsatisfiable == action { @@ -76,14 +96,24 @@ func filterTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint, return nil, err } tsc := topologySpreadConstraint{ - MaxSkew: c.MaxSkew, - TopologyKey: c.TopologyKey, - Selector: selector, - MinDomains: 1, // if MinDomains is nil, we treat MinDomains as 1. + MaxSkew: c.MaxSkew, + TopologyKey: c.TopologyKey, + Selector: selector, + MinDomains: 1, // If MinDomains is nil, we treat MinDomains as 1. + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, // If NodeAffinityPolicy is nil, we treat NodeAffinityPolicy as "Honor". + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, // If NodeTaintsPolicy is nil, we treat NodeTaintsPolicy as "Ignore". } if enableMinDomainsInPodTopologySpread && c.MinDomains != nil { tsc.MinDomains = *c.MinDomains } + if enableNodeInclusionPolicyInPodTopologySpread { + if c.NodeAffinityPolicy != nil { + tsc.NodeAffinityPolicy = *c.NodeAffinityPolicy + } + if c.NodeTaintsPolicy != nil { + tsc.NodeTaintsPolicy = *c.NodeTaintsPolicy + } + } result = append(result, tsc) } } diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go index 2b3580596e2..2f61f688cb2 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go @@ -231,7 +231,12 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod) if len(pod.Spec.TopologySpreadConstraints) > 0 { // We have feature gating in APIServer to strip the spec // so don't need to re-check feature gate, just check length of Constraints. - constraints, err = filterTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints, v1.DoNotSchedule, pl.enableMinDomainsInPodTopologySpread) + constraints, err = filterTopologySpreadConstraints( + pod.Spec.TopologySpreadConstraints, + v1.DoNotSchedule, + pl.enableMinDomainsInPodTopologySpread, + pl.enableNodeInclusionPolicyInPodTopologySpread, + ) if err != nil { return nil, fmt.Errorf("obtaining pod's hard topology spread constraints: %w", err) } @@ -251,8 +256,8 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod) TpPairToMatchNum: make(map[topologyPair]int, sizeHeuristic(len(allNodes), constraints)), } - requiredSchedulingTerm := nodeaffinity.GetRequiredNodeAffinity(pod) tpCountsByNode := make([]map[topologyPair]int, len(allNodes)) + requiredNodeAffinity := nodeaffinity.GetRequiredNodeAffinity(pod) processNode := func(i int) { nodeInfo := allNodes[i] node := nodeInfo.Node() @@ -260,13 +265,15 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod) klog.ErrorS(nil, "Node not found") return } - // In accordance to design, if NodeAffinity or NodeSelector is defined, - // spreading is applied to nodes that pass those filters. - // Ignore parsing errors for backwards compatibility. - match, _ := requiredSchedulingTerm.Match(node) - if !match { - return + + if !pl.enableNodeInclusionPolicyInPodTopologySpread { + // spreading is applied to nodes that pass those filters. + // Ignore parsing errors for backwards compatibility. + if match, _ := requiredNodeAffinity.Match(node); !match { + return + } } + // Ensure current node's labels contains all topologyKeys in 'Constraints'. if !nodeLabelsMatchSpreadConstraints(node.Labels, constraints) { return @@ -274,6 +281,11 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod) tpCounts := make(map[topologyPair]int, len(constraints)) for _, c := range constraints { + if pl.enableNodeInclusionPolicyInPodTopologySpread && + !c.matchNodeInclusionPolicies(pod, node, requiredNodeAffinity) { + continue + } + pair := topologyPair{key: c.TopologyKey, value: node.Labels[c.TopologyKey]} count := countPodsMatchSelector(nodeInfo.Pods, c.Selector, pod.Namespace) tpCounts[pair] = count diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go b/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go index f8e92eb906a..770b36089de 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go @@ -48,7 +48,14 @@ var cmpOpts = []cmp.Option{ }), } -var topologySpreadFunc = frameworkruntime.FactoryAdapter(feature.Features{}, New) +var ( + topologySpreadFunc = frameworkruntime.FactoryAdapter(feature.Features{}, New) + ignorePolicy = v1.NodeInclusionPolicyIgnore + honorPolicy = v1.NodeInclusionPolicyHonor + fooSelector = st.MakeLabelSelector().Exists("foo").Obj() + barSelector = st.MakeLabelSelector().Exists("bar").Obj() + taints = []v1.Taint{{Key: v1.TaintNodeUnschedulable, Value: "", Effect: v1.TaintEffectPreferNoSchedule}} +) func (p *criticalPaths) sort() { if p[0].MatchNum == p[1].MatchNum && p[0].TopologyValue > p[1].TopologyValue { @@ -58,23 +65,23 @@ func (p *criticalPaths) sort() { } func TestPreFilterState(t *testing.T) { - fooSelector := st.MakeLabelSelector().Exists("foo").Obj() - barSelector := st.MakeLabelSelector().Exists("bar").Obj() + tests := []struct { - name string - pod *v1.Pod - nodes []*v1.Node - existingPods []*v1.Pod - enableMinDomains bool - objs []runtime.Object - defaultConstraints []v1.TopologySpreadConstraint - want *preFilterState + name string + pod *v1.Pod + nodes []*v1.Node + existingPods []*v1.Pod + objs []runtime.Object + defaultConstraints []v1.TopologySpreadConstraint + want *preFilterState + enableMinDomains bool + enableNodeInclustionPolicy bool }{ { name: "clean cluster with one spreadConstraint", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 5, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(5, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj(), nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -84,10 +91,12 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 5, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("foo", "bar").Obj()), - MinDomains: 1, + MaxSkew: 5, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("foo", "bar").Obj()), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -101,9 +110,9 @@ func TestPreFilterState(t *testing.T) { }, { name: "normal case with one spreadConstraint", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, fooSelector, nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -120,10 +129,12 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -137,9 +148,9 @@ func TestPreFilterState(t *testing.T) { }, { name: "normal case with one spreadConstraint, on a 3-zone cluster", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -158,10 +169,12 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -176,9 +189,9 @@ func TestPreFilterState(t *testing.T) { }, { name: "namespace mismatch doesn't count", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, fooSelector, nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -195,10 +208,12 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -213,8 +228,8 @@ func TestPreFilterState(t *testing.T) { { name: "normal case with two spreadConstraints", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -234,16 +249,20 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -263,10 +282,10 @@ func TestPreFilterState(t *testing.T) { { name: "soft spreadConstraints should be bypassed", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). - SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -285,16 +304,20 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -313,8 +336,8 @@ func TestPreFilterState(t *testing.T) { { name: "different labelSelectors - simple version", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -328,16 +351,20 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, barSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -356,8 +383,8 @@ func TestPreFilterState(t *testing.T) { { name: "different labelSelectors - complex pods", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -376,16 +403,20 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, barSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -405,8 +436,8 @@ func TestPreFilterState(t *testing.T) { name: "two spreadConstraints, and with podAffinity", pod: st.MakePod().Name("p").Label("foo", ""). NodeAffinityNotIn("node", []string{"node-x"}). // exclude node-x - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -426,16 +457,20 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -465,16 +500,20 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 3, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("foo", "bar").Obj()), - MinDomains: 1, + MaxSkew: 3, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("foo", "bar").Obj()), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 5, - TopologyKey: "rack", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("foo", "bar").Obj()), - MinDomains: 1, + MaxSkew: 5, + TopologyKey: "rack", + Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("foo", "bar").Obj()), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -498,8 +537,9 @@ func TestPreFilterState(t *testing.T) { { name: "default constraints and a service, but pod has constraints", pod: st.MakePod().Name("p").Label("foo", "bar").Label("baz", "tar"). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("baz", "tar").Obj(), nil). - SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("fot", "rok").Obj(), nil).Obj(), + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("baz", "tar").Obj(), nil, nil, nil). + SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("fot", "rok").Obj(), nil, nil, nil). + Obj(), defaultConstraints: []v1.TopologySpreadConstraint{ {MaxSkew: 2, TopologyKey: "node", WhenUnsatisfiable: v1.DoNotSchedule}, }, @@ -509,10 +549,12 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("baz", "tar").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("baz", "tar").Obj()), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -535,8 +577,8 @@ func TestPreFilterState(t *testing.T) { { name: "TpKeyToDomainsNum is calculated when MinDomains is enabled", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -557,16 +599,20 @@ func TestPreFilterState(t *testing.T) { want: &preFilterState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -587,6 +633,552 @@ func TestPreFilterState(t *testing.T) { }, }, }, + { + name: "feature gate disabled with NodeAffinityPolicy", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-a", 1}, {"node-b", 2}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + }, + }, + enableNodeInclustionPolicy: false, + }, + { + name: "NodeAffinityPolicy honored with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-a", 1}, {"node-b", 2}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy ignored with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyIgnore, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-c", 0}, {"node-a", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + {key: "node", value: "node-c"}: 0, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy honored with nodeAffinity", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-e").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-a", 1}, {"node-b", 2}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy ignored with nodeAffinity", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyIgnore, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-c", 0}, {"node-a", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + {key: "node", value: "node-c"}: 0, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "feature gate disabled with NodeTaintsPolicy", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Taints(taints).Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-c", 0}, {"node-a", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + {key: "node", value: "node-c"}: 0, + }, + }, + enableNodeInclustionPolicy: false, + }, + { + name: "NodeTaintsPolicy ignored", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Taints(taints).Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-c", 0}, {"node-a", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + {key: "node", value: "node-c"}: 0, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeTaintsPolicy honored", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, &honorPolicy). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Taints(taints).Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyHonor, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-a", 1}, {"node-b", 2}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeTaintsPolicy honored with tolerated taints", + pod: st.MakePod().Name("p").Label("foo", ""). + Toleration(v1.TaintNodeUnschedulable). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, &honorPolicy). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Taints(taints).Label("bar", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a").Node("node-a").Label("bar", "").Obj(), + st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-c").Node("node-b").Label("bar", "").Obj(), + st.MakePod().Name("p-d").Node("node-c").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyHonor, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "node": {{"node-c", 0}, {"node-a", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "node", value: "node-a"}: 1, + {key: "node", value: "node-b"}: 2, + {key: "node", value: "node-c"}: 0, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "two node inclusion Constraints, zone: honor/ignore, node: ignore/ignore", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Label("foo", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyIgnore, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "zone": {{"zone1", 0}, {"zone2", 1}}, + "node": {{"node-a", 0}, {"node-x", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "zone", value: "zone1"}: 0, + {key: "zone", value: "zone2"}: 1, + {key: "node", value: "node-a"}: 0, + {key: "node", value: "node-b"}: 2, + {key: "node", value: "node-x"}: 1, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "two node inclusion Constraints, zone: honor/honor, node: honor/ignore", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, &honorPolicy). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Taints(taints).Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Label("foo", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyHonor, + }, + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "zone": {{"zone1", 0}, {"zone2", 1}}, + "node": {{"node-a", 0}, {"node-x", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "zone", value: "zone1"}: 0, + {key: "zone", value: "zone2"}: 1, + {key: "node", value: "node-a"}: 0, + {key: "node", value: "node-b"}: 2, + {key: "node", value: "node-x"}: 1, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "two node inclusion Constraints, zone: honor/ignore, node: honor/ignore", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Label("foo", "").Taints(taints).Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "zone": {{"zone1", 0}, {"zone2", 1}}, + "node": {{"node-b", 0}, {"node-x", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "zone", value: "zone1"}: 0, + {key: "zone", value: "zone2"}: 1, + {key: "node", value: "node-b"}: 0, + {key: "node", value: "node-x"}: 1, + }, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "two node inclusion Constraints, zone: ignore/ignore, node: honor/honor", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, &ignorePolicy, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, &honorPolicy). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Label("foo", "").Taints(taints).Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Label("foo", "").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(), + st.MakePod().Name("p-x2").Node("node-x").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyIgnore, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyHonor, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "zone": {{"zone1", 2}, {"zone2", 3}}, + "node": {{"node-y", 1}, {"node-x", 2}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "zone", value: "zone1"}: 2, + {key: "zone", value: "zone2"}: 3, + {key: "node", value: "node-x"}: 2, + {key: "node", value: "node-y"}: 1, + }, + }, + enableNodeInclustionPolicy: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -599,6 +1191,7 @@ func TestPreFilterState(t *testing.T) { p := plugintesting.SetupPluginWithInformers(ctx, t, topologySpreadFunc, args, cache.NewSnapshot(tt.existingPods, tt.nodes), tt.objs) p.(*PodTopologySpread).enableMinDomainsInPodTopologySpread = tt.enableMinDomains + p.(*PodTopologySpread).enableNodeInclusionPolicyInPodTopologySpread = tt.enableNodeInclustionPolicy cs := framework.NewCycleState() if _, s := p.(*PodTopologySpread).PreFilter(ctx, cs, tt.pod); !s.IsSuccess() { @@ -617,10 +1210,12 @@ func TestPreFilterState(t *testing.T) { func TestPreFilterStateAddPod(t *testing.T) { nodeConstraint := topologySpreadConstraint{ - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, } zoneConstraint := nodeConstraint zoneConstraint.TopologyKey = "zone" @@ -636,7 +1231,7 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "node a and b both impact current min match", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: nil, // it's an empty cluster @@ -659,7 +1254,7 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "only node a impacts current min match", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -684,7 +1279,7 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "add a pod in a different namespace doesn't change topologyKeyToMinPodsMap", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-a1").Namespace("ns1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -709,7 +1304,7 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "add pod on non-critical node won't trigger re-calculation", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -734,8 +1329,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "node a and x both impact topologyKeyToMinPodsMap on zone and node", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: nil, // it's an empty cluster @@ -761,8 +1356,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "only node a impacts topologyKeyToMinPodsMap on zone and node", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -790,8 +1385,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "node a impacts topologyKeyToMinPodsMap on node, node x impacts topologyKeyToMinPodsMap on zone", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -823,8 +1418,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "Constraints hold different labelSelectors, node a impacts topologyKeyToMinPodsMap on zone", preemptor: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -842,10 +1437,12 @@ func TestPreFilterStateAddPod(t *testing.T) { Constraints: []topologySpreadConstraint{ zoneConstraint, { - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("bar").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -864,8 +1461,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "Constraints hold different labelSelectors, node a impacts topologyKeyToMinPodsMap on both zone and node", preemptor: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Label("bar", "").Obj(), existingPods: []*v1.Pod{ @@ -883,10 +1480,12 @@ func TestPreFilterStateAddPod(t *testing.T) { Constraints: []topologySpreadConstraint{ zoneConstraint, { - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("bar").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, TpKeyToCriticalPaths: map[string]*criticalPaths{ @@ -933,10 +1532,12 @@ func TestPreFilterStateAddPod(t *testing.T) { func TestPreFilterStateRemovePod(t *testing.T) { nodeConstraint := topologySpreadConstraint{ - MaxSkew: 1, - TopologyKey: "node", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, } zoneConstraint := nodeConstraint zoneConstraint.TopologyKey = "zone" @@ -955,7 +1556,7 @@ func TestPreFilterStateRemovePod(t *testing.T) { // So preemption is triggered. name: "one spreadConstraint on zone, topologyKeyToMinPodsMap unchanged", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -983,7 +1584,7 @@ func TestPreFilterStateRemovePod(t *testing.T) { { name: "one spreadConstraint on node, topologyKeyToMinPodsMap changed", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1013,7 +1614,7 @@ func TestPreFilterStateRemovePod(t *testing.T) { { name: "delete an irrelevant pod won't help", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1044,7 +1645,7 @@ func TestPreFilterStateRemovePod(t *testing.T) { { name: "delete a non-existing pod won't help", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1075,8 +1676,8 @@ func TestPreFilterStateRemovePod(t *testing.T) { { name: "two spreadConstraints", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1154,7 +1755,7 @@ func BenchmarkFilter(b *testing.B) { { name: "1000nodes/single-constraint-zone", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelTopologyZone, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelTopologyZone, v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -1163,7 +1764,7 @@ func BenchmarkFilter(b *testing.B) { { name: "1000nodes/single-constraint-node", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -1172,8 +1773,8 @@ func BenchmarkFilter(b *testing.B) { { name: "1000nodes/two-Constraints-zone-node", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, v1.LabelTopologyZone, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, v1.LabelHostname, v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, v1.LabelTopologyZone, v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, v1.LabelHostname, v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -1219,18 +1820,19 @@ func mustConvertLabelSelectorAsSelector(t *testing.T, ls *metav1.LabelSelector) func TestSingleConstraint(t *testing.T) { tests := []struct { - name string - pod *v1.Pod - nodes []*v1.Node - existingPods []*v1.Pod - enableMinDomains bool - wantStatusCode map[string]framework.Code + name string + pod *v1.Pod + nodes []*v1.Node + existingPods []*v1.Pod + wantStatusCode map[string]framework.Code + enableMinDomains bool + enableNodeInclustionPolicy bool }{ { name: "no existing pods", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1246,9 +1848,9 @@ func TestSingleConstraint(t *testing.T) { }, { name: "no existing pods, incoming pod doesn't match itself", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, barSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1264,9 +1866,9 @@ func TestSingleConstraint(t *testing.T) { }, { name: "existing pods in a different namespace do not count", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1288,9 +1890,9 @@ func TestSingleConstraint(t *testing.T) { }, { name: "pods spread across zones as 3/3, all nodes fit", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1316,9 +1918,9 @@ func TestSingleConstraint(t *testing.T) { // TODO(Huang-Wei): maybe document this to remind users that typos on node labels // can cause unexpected behavior name: "pods spread across zones as 1/2 due to absence of label 'zone' on node-b", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zon", "zone1").Label("node", "node-b").Obj(), @@ -1340,9 +1942,9 @@ func TestSingleConstraint(t *testing.T) { }, { name: "pod cannot be scheduled as all nodes don't have label 'rack'", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "rack", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "rack", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(), @@ -1354,9 +1956,9 @@ func TestSingleConstraint(t *testing.T) { }, { name: "pods spread across nodes as 2/1/0/3, only node-x fits", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1380,9 +1982,9 @@ func TestSingleConstraint(t *testing.T) { }, { name: "pods spread across nodes as 2/1/0/3, maxSkew is 2, node-b and node-x fit", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 2, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(2, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1410,9 +2012,9 @@ func TestSingleConstraint(t *testing.T) { // in this case, placement of the new pod doesn't change pod distribution of the cluster // as the incoming pod doesn't have label "foo" name: "pods spread across nodes as 2/1/0/3, but pod doesn't match itself", - pod: st.MakePod().Name("p").Label("bar", "").SpreadConstraint( - 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("bar", ""). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1437,13 +2039,13 @@ func TestSingleConstraint(t *testing.T) { { // only node-a and node-y are considered, so pods spread as 2/~1~/~0~/3 // ps: '~num~' is a markdown symbol to denote a crossline through 'num' - // but in this unit test, we don't run NodeAffinity Predicate, so node-b and node-x are + // but in this unit test, we don't run NodeAffinity/TaintToleration Predicate, so node-b and node-x are // still expected to be fits; // the fact that node-a fits can prove the underlying logic works name: "incoming pod has nodeAffinity, pods spread as 2/~1~/~0~/3, hence node-a fits", pod: st.MakePod().Name("p").Label("foo", ""). NodeAffinityIn("node", []string{"node-a", "node-y"}). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1468,9 +2070,9 @@ func TestSingleConstraint(t *testing.T) { }, { name: "terminating Pods should be excluded", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), @@ -1489,7 +2091,7 @@ func TestSingleConstraint(t *testing.T) { name: "incoming pod has nodeAffinity, pods spread as 0/~2~/0/1, hence node-a fits", pod: st.MakePod().Name("p").Label("foo", ""). NodeAffinityNotIn("node", []string{"node-b"}). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1515,8 +2117,10 @@ func TestSingleConstraint(t *testing.T) { 2, "node", v1.DoNotSchedule, - st.MakeLabelSelector().Exists("foo").Obj(), + fooSelector, pointer.Int32(4), // larger than the number of domains(3) + nil, + nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), @@ -1543,8 +2147,10 @@ func TestSingleConstraint(t *testing.T) { 2, "node", v1.DoNotSchedule, - st.MakeLabelSelector().Exists("foo").Obj(), + fooSelector, pointer.Int32(2), // smaller than the number of domains(3) + nil, + nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), @@ -1567,13 +2173,15 @@ func TestSingleConstraint(t *testing.T) { }, { name: "pods spread across zone as 2/1, maxSkew is 2 and the number of domains < minDomains, then the third and fourth nodes fit", - pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(2, - "zone", - v1.DoNotSchedule, - st.MakeLabelSelector().Exists("foo").Obj(), - pointer.Int32(3), // larger than the number of domains(2) - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( + 2, + "zone", + v1.DoNotSchedule, + fooSelector, + pointer.Int32(3), // larger than the number of domains(2) + nil, + nil, + ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1595,13 +2203,15 @@ func TestSingleConstraint(t *testing.T) { }, { name: "pods spread across zone as 2/1, maxSkew is 2 and the number of domains > minDomains, then the all nodes fit", - pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(2, - "zone", - v1.DoNotSchedule, - st.MakeLabelSelector().Exists("foo").Obj(), - pointer.Int32(1), // smaller than the number of domains(2) - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( + 2, + "zone", + v1.DoNotSchedule, + fooSelector, + pointer.Int32(1), // smaller than the number of domains(2) + nil, + nil, + ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), @@ -1621,6 +2231,160 @@ func TestSingleConstraint(t *testing.T) { "node-y": framework.Success, }, }, + { + // pods spread across node as 1/1/0/~0~ + name: "NodeAffinityPolicy honored with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-x").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Success, // in real case, when we disable NodeAffinity Plugin, node-y will be success. + }, + enableNodeInclustionPolicy: true, + }, + { + // pods spread across node as 1/1/0/~1~ + name: "NodeAffinityPolicy ignored with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-x").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Unschedulable, + }, + enableNodeInclustionPolicy: true, + }, + { + // pods spread across node as 1/1/0/~0~ + name: "NodeAffinityPolicy honored with nodeAffinity", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-x").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Success, // in real case, when we disable NodeAffinity Plugin, node-y will be success. + }, + enableNodeInclustionPolicy: true, + }, + { + // pods spread across node as 1/1/0/~1~ + name: "NodeAffinityPolicy ignored with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-x").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Unschedulable, + }, + enableNodeInclustionPolicy: true, + }, + { + // pods spread across node as 1/1/0/~0~ + name: "NodeTaintsPolicy honored", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, &honorPolicy). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-x").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Taints(taints).Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Success, // in real case, when we disable TaintToleration Plugin, node-y will be success. + }, + enableNodeInclustionPolicy: true, + }, + { + // pods spread across node as 1/1/0/~1~ + name: "NodeTaintsPolicy ignored", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-x").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("node", "node-y").Taints(taints).Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Unschedulable, + }, + enableNodeInclustionPolicy: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1628,6 +2392,7 @@ func TestSingleConstraint(t *testing.T) { pl := plugintesting.SetupPlugin(t, topologySpreadFunc, &config.PodTopologySpreadArgs{DefaultingType: config.ListDefaulting}, snapshot) p := pl.(*PodTopologySpread) p.enableMinDomainsInPodTopologySpread = tt.enableMinDomains + p.enableNodeInclusionPolicyInPodTopologySpread = tt.enableNodeInclustionPolicy state := framework.NewCycleState() if _, s := p.PreFilter(context.Background(), state, tt.pod); !s.IsSuccess() { t.Errorf("preFilter failed with status: %v", s) @@ -1646,11 +2411,12 @@ func TestSingleConstraint(t *testing.T) { func TestMultipleConstraints(t *testing.T) { tests := []struct { - name string - pod *v1.Pod - nodes []*v1.Node - existingPods []*v1.Pod - wantStatusCode map[string]framework.Code + name string + pod *v1.Pod + nodes []*v1.Node + existingPods []*v1.Pod + wantStatusCode map[string]framework.Code + enableNodeInclustionPolicy bool }{ { // 1. to fulfil "zone" constraint, incoming pod can be placed on any zone (hence any node) @@ -1658,8 +2424,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-x name: "two Constraints on zone and node, spreads = [3/3, 2/1/0/3]", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1688,8 +2454,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns no node name: "two Constraints on zone and node, spreads = [3/4, 2/1/0/4]", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1719,8 +2485,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-x name: "Constraints hold different labelSelectors, spreads = [1/0, 1/0/0/1]", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1745,8 +2511,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns no node name: "Constraints hold different labelSelectors, spreads = [1/0, 0/0/1/1]", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1772,8 +2538,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-b name: "Constraints hold different labelSelectors, spreads = [2/3, 1/0/0/1]", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1801,8 +2567,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-a and node-b name: "Constraints hold different labelSelectors but pod doesn't match itself on 'zone' constraint", pod: st.MakePod().Name("p").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1828,8 +2594,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-b name: "two Constraints on zone and node, absence of label 'node' on node-x, spreads = [1/1, 1/0/0/1]", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1848,12 +2614,128 @@ func TestMultipleConstraints(t *testing.T) { "node-y": framework.Unschedulable, }, }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as 2/~0~ + // 2. to fulfil "node" constraint, pods spread across zones as 1/1/0/~1~ + // intersection of (1) and (2) returns node-x + name: "two node inclusion Constraints, zone: honor/ignore, node: ignore/ignore", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Unschedulable, + }, + enableNodeInclustionPolicy: true, + }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as 2/0 + // 2. to fulfil "node" constraint, pods spread across zones as 1/1/0/~1~ + // intersection of (1) and (2) returns node-x + name: "two node inclusion Constraints, zone: honor/honor, node: honor/ignore", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, &honorPolicy). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Taints(taints).Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Unschedulable, + }, + enableNodeInclustionPolicy: true, + }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as 1/~1~ + // 2. to fulfil "node" constraint, pods spread across zones as 1/0/0/~1~ + // intersection of (1) and (2) returns node-x + name: "two node inclusion Constraints, zone: honor/ignore, node: honor/ignore", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Label("foo", "").Taints(taints).Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Success, + "node-x": framework.Success, + "node-y": framework.Unschedulable, + }, + enableNodeInclustionPolicy: true, + }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as 1/0 + // 2. to fulfil "node" constraint, pods spread across zones as 1/~1~/0/~1~ + // intersection of (1) and (2) returns node-x + name: "two node inclusion Constraints, zone: honor/honor, node: ignore/ignore", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil, nil, &honorPolicy). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Label("foo", "").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Label("foo", "").Taints(taints).Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Unschedulable, + }, + enableNodeInclustionPolicy: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { snapshot := cache.NewSnapshot(tt.existingPods, tt.nodes) pl := plugintesting.SetupPlugin(t, topologySpreadFunc, &config.PodTopologySpreadArgs{DefaultingType: config.ListDefaulting}, snapshot) p := pl.(*PodTopologySpread) + p.enableNodeInclusionPolicyInPodTopologySpread = tt.enableNodeInclustionPolicy state := framework.NewCycleState() if _, s := p.PreFilter(context.Background(), state, tt.pod); !s.IsSuccess() { t.Errorf("preFilter failed with status: %v", s) diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/plugin.go b/pkg/scheduler/framework/plugins/podtopologyspread/plugin.go index 2584bc89a2b..a5d17ca5cea 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/plugin.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/plugin.go @@ -54,15 +54,16 @@ var systemDefaultConstraints = []v1.TopologySpreadConstraint{ // PodTopologySpread is a plugin that ensures pod's topologySpreadConstraints is satisfied. type PodTopologySpread struct { - systemDefaulted bool - parallelizer parallelize.Parallelizer - defaultConstraints []v1.TopologySpreadConstraint - sharedLister framework.SharedLister - services corelisters.ServiceLister - replicationCtrls corelisters.ReplicationControllerLister - replicaSets appslisters.ReplicaSetLister - statefulSets appslisters.StatefulSetLister - enableMinDomainsInPodTopologySpread bool + systemDefaulted bool + parallelizer parallelize.Parallelizer + defaultConstraints []v1.TopologySpreadConstraint + sharedLister framework.SharedLister + services corelisters.ServiceLister + replicationCtrls corelisters.ReplicationControllerLister + replicaSets appslisters.ReplicaSetLister + statefulSets appslisters.StatefulSetLister + enableMinDomainsInPodTopologySpread bool + enableNodeInclusionPolicyInPodTopologySpread bool } var _ framework.PreFilterPlugin = &PodTopologySpread{} @@ -98,6 +99,7 @@ func New(plArgs runtime.Object, h framework.Handle, fts feature.Features) (frame sharedLister: h.SnapshotSharedLister(), defaultConstraints: args.DefaultConstraints, enableMinDomainsInPodTopologySpread: fts.EnableMinDomainsInPodTopologySpread, + enableNodeInclusionPolicyInPodTopologySpread: fts.EnableNodeInclusionPolicyInPodTopologySpread, } if args.DefaultingType == config.SystemDefaulting { pl.defaultConstraints = systemDefaultConstraints diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go b/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go index 7b71f786b1d..1cb7355e2a7 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go @@ -59,7 +59,12 @@ func (s *preScoreState) Clone() framework.StateData { func (pl *PodTopologySpread) initPreScoreState(s *preScoreState, pod *v1.Pod, filteredNodes []*v1.Node, requireAllTopologies bool) error { var err error if len(pod.Spec.TopologySpreadConstraints) > 0 { - s.Constraints, err = filterTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints, v1.ScheduleAnyway, pl.enableMinDomainsInPodTopologySpread) + s.Constraints, err = filterTopologySpreadConstraints( + pod.Spec.TopologySpreadConstraints, + v1.ScheduleAnyway, + pl.enableMinDomainsInPodTopologySpread, + pl.enableNodeInclusionPolicyInPodTopologySpread, + ) if err != nil { return fmt.Errorf("obtaining pod's soft topology spread constraints: %w", err) } @@ -148,14 +153,25 @@ func (pl *PodTopologySpread) PreScore( if node == nil { return } - // (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity - // (2) All topologyKeys need to be present in `node` - match, _ := requiredNodeAffinity.Match(node) - if !match || (requireAllTopologies && !nodeLabelsMatchSpreadConstraints(node.Labels, state.Constraints)) { + + if !pl.enableNodeInclusionPolicyInPodTopologySpread { + // `node` should satisfy incoming pod's NodeSelector/NodeAffinity + if match, _ := requiredNodeAffinity.Match(node); !match { + return + } + } + + // All topologyKeys need to be present in `node` + if requireAllTopologies && !nodeLabelsMatchSpreadConstraints(node.Labels, state.Constraints) { return } for _, c := range state.Constraints { + if pl.enableNodeInclusionPolicyInPodTopologySpread && + !c.matchNodeInclusionPolicies(pod, node, requiredNodeAffinity) { + continue + } + pair := topologyPair{key: c.TopologyKey, value: node.Labels[c.TopologyKey]} // If current topology pair is not associated with any candidate node, // continue to avoid unnecessary calculation. diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go b/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go index b08a7d1446f..b9089957a51 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go @@ -18,6 +18,7 @@ package podtopologyspread import ( "context" + "fmt" "testing" "github.com/google/go-cmp/cmp" @@ -42,18 +43,19 @@ var podTopologySpreadFunc = frameworkruntime.FactoryAdapter(feature.Features{}, func TestPreScoreStateEmptyNodes(t *testing.T) { tests := []struct { - name string - pod *v1.Pod - nodes []*v1.Node - objs []runtime.Object - config config.PodTopologySpreadArgs - want *preScoreState + name string + pod *v1.Pod + nodes []*v1.Node + objs []runtime.Object + config config.PodTopologySpreadArgs + want *preScoreState + enableNodeInclustionPolicy bool }{ { name: "normal case", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), @@ -66,16 +68,20 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { want: &preScoreState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 1, - TopologyKey: v1.LabelHostname, - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: v1.LabelHostname, + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, IgnoredNodes: sets.NewString(), @@ -89,8 +95,8 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { { name: "node-x doesn't have label zone", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), @@ -103,16 +109,20 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { want: &preScoreState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: "zone", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 1, - TopologyKey: v1.LabelHostname, - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("bar").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: v1.LabelHostname, + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, IgnoredNodes: sets.NewString("node-x"), @@ -136,21 +146,25 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { st.MakeNode().Name("node-d").Label(v1.LabelHostname, "node-d").Obj(), }, objs: []runtime.Object{ - &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}}, + &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: fooSelector}}, }, want: &preScoreState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 3, - TopologyKey: v1.LabelHostname, - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 3, + TopologyKey: v1.LabelHostname, + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 5, - TopologyKey: v1.LabelTopologyZone, - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 5, + TopologyKey: v1.LabelTopologyZone, + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, IgnoredNodes: sets.NewString(), @@ -166,8 +180,15 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { pod: st.MakePod().Name("p").Namespace("default").Label("foo", "tar").Label("baz", "sup").OwnerReference("rs1", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")).Obj(), config: config.PodTopologySpreadArgs{ DefaultConstraints: []v1.TopologySpreadConstraint{ - {MaxSkew: 1, TopologyKey: v1.LabelHostname, WhenUnsatisfiable: v1.ScheduleAnyway}, - {MaxSkew: 2, TopologyKey: "rack", WhenUnsatisfiable: v1.DoNotSchedule}, + { + MaxSkew: 1, + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.ScheduleAnyway, + }, + {MaxSkew: 2, + TopologyKey: "rack", + WhenUnsatisfiable: v1.DoNotSchedule, + }, {MaxSkew: 2, TopologyKey: "planet", WhenUnsatisfiable: v1.ScheduleAnyway}, }, DefaultingType: config.ListDefaulting, @@ -176,21 +197,25 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { st.MakeNode().Name("node-a").Label("rack", "rack1").Label(v1.LabelHostname, "node-a").Label("planet", "mars").Obj(), }, objs: []runtime.Object{ - &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}}, + &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: fooSelector}}, }, want: &preScoreState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 1, - TopologyKey: v1.LabelHostname, - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 1, + TopologyKey: v1.LabelHostname, + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, { - MaxSkew: 2, - TopologyKey: "planet", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()), - MinDomains: 1, + MaxSkew: 2, + TopologyKey: "planet", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, IgnoredNodes: sets.NewString(), @@ -205,7 +230,11 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { pod: st.MakePod().Name("p").Namespace("default").Label("foo", "bar").Label("baz", "sup").OwnerReference("rs2", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")).Obj(), config: config.PodTopologySpreadArgs{ DefaultConstraints: []v1.TopologySpreadConstraint{ - {MaxSkew: 2, TopologyKey: "planet", WhenUnsatisfiable: v1.ScheduleAnyway}, + { + MaxSkew: 2, + TopologyKey: "planet", + WhenUnsatisfiable: v1.ScheduleAnyway, + }, }, DefaultingType: config.ListDefaulting, }, @@ -223,11 +252,16 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { name: "default constraints and a replicaset, but pod has constraints", pod: st.MakePod().Name("p").Namespace("default").Label("foo", "bar").Label("baz", "sup"). OwnerReference("rs1", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj(), nil). - SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("baz", "sup").Obj(), nil).Obj(), + SpreadConstraint(1, "zone", v1.DoNotSchedule, barSelector, nil, nil, nil). + SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("baz", "sup").Obj(), nil, nil, nil). + Obj(), config: config.PodTopologySpreadArgs{ DefaultConstraints: []v1.TopologySpreadConstraint{ - {MaxSkew: 2, TopologyKey: "galaxy", WhenUnsatisfiable: v1.ScheduleAnyway}, + { + MaxSkew: 2, + TopologyKey: "galaxy", + WhenUnsatisfiable: v1.ScheduleAnyway, + }, }, DefaultingType: config.ListDefaulting, }, @@ -235,15 +269,17 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { st.MakeNode().Name("node-a").Label("planet", "mars").Label("galaxy", "andromeda").Obj(), }, objs: []runtime.Object{ - &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}}, + &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: fooSelector}}, }, want: &preScoreState{ Constraints: []topologySpreadConstraint{ { - MaxSkew: 2, - TopologyKey: "planet", - Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("baz", "sup").Obj()), - MinDomains: 1, + MaxSkew: 2, + TopologyKey: "planet", + Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("baz", "sup").Obj()), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, }, }, IgnoredNodes: sets.NewString(), @@ -253,6 +289,208 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { TopologyNormalizingWeight: []float64{topologyNormalizingWeight(1)}, }, }, + { + name: "NodeAffinityPolicy honored with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), + }, + config: config.PodTopologySpreadArgs{ + DefaultingType: config.ListDefaulting, + }, + want: &preScoreState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + IgnoredNodes: sets.NewString(), + TopologyPairToPodCounts: map[topologyPair]*int64{ + {key: "zone", value: "zone1"}: pointer.Int64Ptr(0), + {key: "zone", value: "zone2"}: pointer.Int64Ptr(0), + }, + TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy ignored with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), + }, + config: config.PodTopologySpreadArgs{ + DefaultingType: config.ListDefaulting, + }, + want: &preScoreState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyIgnore, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + IgnoredNodes: sets.NewString(), + TopologyPairToPodCounts: map[topologyPair]*int64{ + {key: "zone", value: "zone1"}: pointer.Int64Ptr(0), + {key: "zone", value: "zone2"}: pointer.Int64Ptr(0), + }, + TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy honored with nodeAffinity", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), + }, + config: config.PodTopologySpreadArgs{ + DefaultingType: config.ListDefaulting, + }, + want: &preScoreState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + IgnoredNodes: sets.NewString(), + TopologyPairToPodCounts: map[topologyPair]*int64{ + {key: "zone", value: "zone1"}: pointer.Int64Ptr(0), + {key: "zone", value: "zone2"}: pointer.Int64Ptr(0), + }, + TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy ignored with nodeAffinity", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(), + }, + config: config.PodTopologySpreadArgs{ + DefaultingType: config.ListDefaulting, + }, + want: &preScoreState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyIgnore, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + IgnoredNodes: sets.NewString(), + TopologyPairToPodCounts: map[topologyPair]*int64{ + {key: "zone", value: "zone1"}: pointer.Int64Ptr(0), + {key: "zone", value: "zone2"}: pointer.Int64Ptr(0), + }, + TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeTaintsPolicy honored", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, nil, &honorPolicy). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Taints(taints).Obj(), + }, + config: config.PodTopologySpreadArgs{ + DefaultingType: config.ListDefaulting, + }, + want: &preScoreState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyHonor, + }, + }, + IgnoredNodes: sets.NewString(), + TopologyPairToPodCounts: map[topologyPair]*int64{ + {key: "zone", value: "zone1"}: pointer.Int64Ptr(0), + {key: "zone", value: "zone2"}: pointer.Int64Ptr(0), + }, + TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeTaintsPolicy ignored", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Taints(taints).Obj(), + }, + config: config.PodTopologySpreadArgs{ + DefaultingType: config.ListDefaulting, + }, + want: &preScoreState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, barSelector), + MinDomains: 1, + NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, + NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, + }, + }, + IgnoredNodes: sets.NewString(), + TopologyPairToPodCounts: map[topologyPair]*int64{ + {key: "zone", value: "zone1"}: pointer.Int64Ptr(0), + {key: "zone", value: "zone2"}: pointer.Int64Ptr(0), + }, + TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)}, + }, + enableNodeInclustionPolicy: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -265,7 +503,7 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { if err != nil { t.Fatalf("Failed creating framework runtime: %v", err) } - pl, err := New(&tt.config, f, feature.Features{}) + pl, err := New(&tt.config, f, feature.Features{EnableNodeInclusionPolicyInPodTopologySpread: tt.enableNodeInclustionPolicy}) if err != nil { t.Fatalf("Failed creating plugin: %v", err) } @@ -290,25 +528,25 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { func TestPodTopologySpreadScore(t *testing.T) { tests := []struct { - name string - pod *v1.Pod - existingPods []*v1.Pod - nodes []*v1.Node - failedNodes []*v1.Node // nodes + failedNodes = all nodes - objs []runtime.Object - want framework.NodeScoreList + name string + pod *v1.Pod + existingPods []*v1.Pod + nodes []*v1.Node + failedNodes []*v1.Node // nodes + failedNodes = all nodes + objs []runtime.Object + want framework.NodeScoreList + enableNodeInclustionPolicy bool }{ // Explanation on the Legend: // a) X/Y means there are X matching pods on node1 and Y on node2, both nodes are candidates // (i.e. they have passed all predicates) // b) X/~Y~ means there are X matching pods on node1 and Y on node2, but node Y is NOT a candidate // c) X/?Y? means there are X matching pods on node1 and Y on node2, both nodes are candidates - // but node2 either i) doesn't have all required topologyKeys present, or ii) doesn't match - // incoming pod's nodeSelector/nodeAffinity + // but node2 doesn't have all required topologyKeys present. { name: "one constraint on node, no existing pods", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), @@ -323,7 +561,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // if there is only one candidate node, it should be scored to 100 name: "one constraint on node, only one node is candidate", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -343,7 +581,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "one constraint on node, all nodes have the same number of matching pods", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -362,7 +600,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 2/1/0/3. name: "one constraint on node, all 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -389,7 +627,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "one constraint on node, all 4 nodes are candidates, maxSkew=2", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(2, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(2, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), // matching pods spread as 2/1/0/3. existingPods: []*v1.Pod{ @@ -417,7 +655,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "one constraint on node, all 4 nodes are candidates, maxSkew=3", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(3, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(3, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ // matching pods spread as 4/3/2/1. @@ -484,7 +722,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 4/2/1/~3~ (node4 is not a candidate) name: "one constraint on node, 3 out of 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -516,7 +754,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 4/?2?/1/~3~, total = 4+?+1 = 5 (as node2 is problematic) name: "one constraint on node, 3 out of 4 nodes are candidates, one node doesn't match topology key", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -548,7 +786,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 4/2/1/~3~ name: "one constraint on zone, 3 out of 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -580,8 +818,8 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 2/~1~/2/~4~. name: "two Constraints on zone and node, 2 out of 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -620,8 +858,8 @@ func TestPodTopologySpreadScore(t *testing.T) { // For the second constraint (node): the matching pods spread as 0/1/0/1 name: "two Constraints on zone and node, with different labelSelectors", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -648,8 +886,8 @@ func TestPodTopologySpreadScore(t *testing.T) { // For the second constraint (node): the matching pods spread as 0/1/0/1 name: "two Constraints on zone and node, with different labelSelectors, some nodes have 0 pods", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-b1").Node("node-b").Label("bar", "").Obj(), @@ -675,8 +913,8 @@ func TestPodTopologySpreadScore(t *testing.T) { // For the second constraint (node): the matching pods spread as 0/1/0/~1~ name: "two Constraints on zone and node, with different labelSelectors, 3 out of 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -701,7 +939,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "existing pods in a different namespace do not count", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Namespace("ns1").Node("node-a").Label("foo", "").Obj(), @@ -720,9 +958,9 @@ func TestPodTopologySpreadScore(t *testing.T) { }, { name: "terminating Pods should be excluded", - pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil, - ).Obj(), + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). + Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(), @@ -744,8 +982,10 @@ func TestPodTopologySpreadScore(t *testing.T) { 2, "node", v1.ScheduleAnyway, - st.MakeLabelSelector().Exists("foo").Obj(), + fooSelector, pointer.Int32(10), // larger than the number of domains(3) + nil, + nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), @@ -765,6 +1005,148 @@ func TestPodTopologySpreadScore(t *testing.T) { {Name: "node-c", Score: 100}, }, }, + { + name: "NodeAffinityPolicy honoed with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + want: []framework.NodeScore{ + {Name: "node-a", Score: 0}, + {Name: "node-b", Score: 33}, + {Name: "node-c", Score: 100}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy ignored with labelSelectors", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + want: []framework.NodeScore{ + {Name: "node-a", Score: 66}, + {Name: "node-b", Score: 100}, + {Name: "node-c", Score: 100}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy honoed with nodeAffinity", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + want: []framework.NodeScore{ + {Name: "node-a", Score: 0}, + {Name: "node-b", Score: 33}, + {Name: "node-c", Score: 100}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy ignored with nodeAffinity", + pod: st.MakePod().Name("p").Label("foo", ""). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, &ignorePolicy, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + want: []framework.NodeScore{ + {Name: "node-a", Score: 66}, + {Name: "node-b", Score: 100}, + {Name: "node-c", Score: 100}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeTaintsPolicy honored", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, &honorPolicy). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Taints(taints).Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + want: []framework.NodeScore{ + {Name: "node-a", Score: 0}, + {Name: "node-b", Score: 33}, + {Name: "node-c", Score: 100}, + }, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeTaintsPolicy ignored", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Taints(taints).Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + want: []framework.NodeScore{ + {Name: "node-a", Score: 66}, + {Name: "node-b", Score: 100}, + {Name: "node-c", Score: 100}, + }, + enableNodeInclustionPolicy: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -775,6 +1157,7 @@ func TestPodTopologySpreadScore(t *testing.T) { state := framework.NewCycleState() pl := plugintesting.SetupPluginWithInformers(ctx, t, podTopologySpreadFunc, &config.PodTopologySpreadArgs{DefaultingType: config.SystemDefaulting}, cache.NewSnapshot(tt.existingPods, allNodes), tt.objs) p := pl.(*PodTopologySpread) + p.enableNodeInclusionPolicyInPodTopologySpread = tt.enableNodeInclustionPolicy status := p.PreScore(context.Background(), state, tt.pod, tt.nodes) if !status.IsSuccess() { @@ -785,12 +1168,14 @@ func TestPodTopologySpreadScore(t *testing.T) { for _, n := range tt.nodes { nodeName := n.Name score, status := p.Score(ctx, state, tt.pod, nodeName) + fmt.Println("get score", score) if !status.IsSuccess() { t.Errorf("unexpected error: %v", status) } gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score}) } + fmt.Println(gotList) status = p.NormalizeScore(ctx, state, tt.pod, gotList) if !status.IsSuccess() { t.Errorf("unexpected error: %v", status) @@ -813,7 +1198,7 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) { { name: "1000nodes/single-constraint-zone", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -822,7 +1207,7 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) { { name: "1000nodes/single-constraint-node", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -831,8 +1216,8 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) { { name: "1000nodes/two-Constraints-zone-node", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). + SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, fooSelector, nil, nil, nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, diff --git a/pkg/scheduler/framework/plugins/registry.go b/pkg/scheduler/framework/plugins/registry.go index 330e0924d11..bc0654fe6fc 100644 --- a/pkg/scheduler/framework/plugins/registry.go +++ b/pkg/scheduler/framework/plugins/registry.go @@ -45,10 +45,11 @@ import ( // through the WithFrameworkOutOfTreeRegistry option. func NewInTreeRegistry() runtime.Registry { fts := plfeature.Features{ - EnablePodDisruptionBudget: feature.DefaultFeatureGate.Enabled(features.PodDisruptionBudget), - EnableReadWriteOncePod: feature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod), - EnableVolumeCapacityPriority: feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority), - EnableMinDomainsInPodTopologySpread: feature.DefaultFeatureGate.Enabled(features.MinDomainsInPodTopologySpread), + EnablePodDisruptionBudget: feature.DefaultFeatureGate.Enabled(features.PodDisruptionBudget), + EnableReadWriteOncePod: feature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod), + EnableVolumeCapacityPriority: feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority), + EnableMinDomainsInPodTopologySpread: feature.DefaultFeatureGate.Enabled(features.MinDomainsInPodTopologySpread), + EnableNodeInclusionPolicyInPodTopologySpread: feature.DefaultFeatureGate.Enabled(features.NodeInclusionPolicyInPodTopologySpread), } return runtime.Registry{ diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go index 6cb7cf64ce9..8bc6a24dcf6 100644 --- a/pkg/scheduler/schedule_one_test.go +++ b/pkg/scheduler/schedule_one_test.go @@ -1754,7 +1754,7 @@ func TestSchedulerSchedulePod(t *testing.T) { Operator: metav1.LabelSelectorOpExists, }, }, - }, nil).Obj(), + }, nil, nil, nil).Obj(), pods: []*v1.Pod{ st.MakePod().Name("pod1").UID("pod1").Label("foo", "").Node("node1").Phase(v1.PodRunning).Obj(), }, @@ -1781,7 +1781,7 @@ func TestSchedulerSchedulePod(t *testing.T) { Operator: metav1.LabelSelectorOpExists, }, }, - }, nil).Obj(), + }, nil, nil, nil).Obj(), pods: []*v1.Pod{ st.MakePod().Name("pod1a").UID("pod1a").Label("foo", "").Node("node1").Phase(v1.PodRunning).Obj(), st.MakePod().Name("pod1b").UID("pod1b").Label("foo", "").Node("node1").Phase(v1.PodRunning).Obj(), diff --git a/pkg/scheduler/testing/wrappers.go b/pkg/scheduler/testing/wrappers.go index 160111d1244..fe84987e439 100644 --- a/pkg/scheduler/testing/wrappers.go +++ b/pkg/scheduler/testing/wrappers.go @@ -547,13 +547,15 @@ func (p *PodWrapper) PodAntiAffinityNotIn(labelKey, topologyKey string, vals []s // SpreadConstraint constructs a TopologySpreadConstraint object and injects // into the inner pod. -func (p *PodWrapper) SpreadConstraint(maxSkew int, tpKey string, mode v1.UnsatisfiableConstraintAction, selector *metav1.LabelSelector, minDomains *int32) *PodWrapper { +func (p *PodWrapper) SpreadConstraint(maxSkew int, tpKey string, mode v1.UnsatisfiableConstraintAction, selector *metav1.LabelSelector, minDomains *int32, nodeAffinityPolicy, nodeTaintsPolicy *v1.NodeInclusionPolicy) *PodWrapper { c := v1.TopologySpreadConstraint{ - MaxSkew: int32(maxSkew), - TopologyKey: tpKey, - WhenUnsatisfiable: mode, - LabelSelector: selector, - MinDomains: minDomains, + MaxSkew: int32(maxSkew), + TopologyKey: tpKey, + WhenUnsatisfiable: mode, + LabelSelector: selector, + MinDomains: minDomains, + NodeAffinityPolicy: nodeAffinityPolicy, + NodeTaintsPolicy: nodeTaintsPolicy, } p.Spec.TopologySpreadConstraints = append(p.Spec.TopologySpreadConstraints, c) return p diff --git a/test/integration/scheduler/filters/filters_test.go b/test/integration/scheduler/filters/filters_test.go index 76cfd922187..7428938e03b 100644 --- a/test/integration/scheduler/filters/filters_test.go +++ b/test/integration/scheduler/filters/filters_test.go @@ -54,6 +54,12 @@ var ( const pollInterval = 100 * time.Millisecond +var ( + ignorePolicy = v1.NodeInclusionPolicyIgnore + honorPolicy = v1.NodeInclusionPolicyHonor + taints = []v1.Taint{{Key: v1.TaintNodeUnschedulable, Value: "", Effect: v1.TaintEffectPreferNoSchedule}} +) + // TestInterPodAffinity verifies that scheduler's inter pod affinity and // anti-affinity predicate functions works correctly. func TestInterPodAffinity(t *testing.T) { @@ -1053,19 +1059,29 @@ func TestInterPodAffinityWithNamespaceSelector(t *testing.T) { // TestPodTopologySpreadFilter verifies that EvenPodsSpread predicate functions well. func TestPodTopologySpreadFilter(t *testing.T) { pause := imageutils.GetPauseImageName() + // default nodes with labels "zone: zone-{0,1}" and "node: ". + defaultNodes := []*v1.Node{ + st.MakeNode().Name("node-0").Label("node", "node-0").Label("zone", "zone-0").Obj(), + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-0").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-1").Obj(), + } + tests := []struct { - name string - incomingPod *v1.Pod - existingPods []*v1.Pod - fits bool - candidateNodes []string // nodes expected to schedule onto - enableMinDomains bool + name string + incomingPod *v1.Pod + existingPods []*v1.Pod + fits bool + nodes []*v1.Node + candidateNodes []string // nodes expected to schedule onto + enableMinDomains bool + enableNodeInclustionPolicy bool }{ // note: naming starts at index 0 { name: "place pod on a 1/1/0/1 cluster with MaxSkew=1, node-2 is the only fit", incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), @@ -1073,12 +1089,13 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-2"}, }, { name: "place pod on a 2/0/0/1 cluster with MaxSkew=2, node-{1,2,3} are good fits", incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). - SpreadConstraint(2, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(2, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(), @@ -1086,26 +1103,28 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-1", "node-2", "node-3"}, }, { name: "pod is required to be placed on zone0, so only node-1 fits", incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). NodeAffinityIn("zone", []string{"zone-0"}). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-1"}, }, { name: "two constraints: pod can only be placed to zone-1/node-2", incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). - SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), @@ -1114,14 +1133,15 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-2"}, }, { name: "pod cannot be placed onto any node", incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster - SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), @@ -1130,14 +1150,15 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, - fits: false, + fits: false, + nodes: defaultNodes, }, { name: "high priority pod can preempt others", incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).Priority(100). NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster - SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().ZeroTerminationGracePeriod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), @@ -1147,6 +1168,7 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-1", "node-2", "node-3"}, }, { @@ -1159,6 +1181,8 @@ func TestPodTopologySpreadFilter(t *testing.T) { hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), pointer.Int32(4), // larger than the number of domains (= 3) + nil, + nil, ). Obj(), existingPods: []*v1.Pod{ @@ -1169,6 +1193,7 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-3"}, enableMinDomains: true, }, @@ -1182,6 +1207,8 @@ func TestPodTopologySpreadFilter(t *testing.T) { hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), pointer.Int32(2), // smaller than the number of domains (= 3) + nil, + nil, ). Obj(), existingPods: []*v1.Pod{ @@ -1192,6 +1219,7 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-1", "node-2", "node-3"}, enableMinDomains: true, }, @@ -1204,6 +1232,8 @@ func TestPodTopologySpreadFilter(t *testing.T) { v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), pointer.Int32(3), // larger than the number of domains(2) + nil, + nil, ).Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p1a").Node("node-0").Label("foo", "").Container(pause).Obj(), @@ -1211,6 +1241,7 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().Name("p3a").Node("node-2").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-2", "node-3"}, enableMinDomains: true, }, @@ -1223,6 +1254,8 @@ func TestPodTopologySpreadFilter(t *testing.T) { v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), pointer.Int32(1), // smaller than the number of domains(2) + nil, + nil, ).Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), @@ -1230,24 +1263,186 @@ func TestPodTopologySpreadFilter(t *testing.T) { st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, + nodes: defaultNodes, candidateNodes: []string{"node-0", "node-1", "node-2", "node-3"}, enableMinDomains: true, }, + { + name: "NodeAffinityPolicy honored with labelSelectors, pods spread across zone as 2/1", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), + }, + candidateNodes: []string{"node-4"}, // node-3 is filtered out by NodeAffinity plugin + enableNodeInclustionPolicy: true, + }, + { + name: "NodeAffinityPolicy ignored with nodeAffinity, pods spread across zone as 1/~2~", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeAffinityIn("foo", []string{""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), + }, + candidateNodes: []string{"node-1", "node-2"}, + enableNodeInclustionPolicy: true, + }, + { + name: "NodeTaintsPolicy honored, pods spread across zone as 2/1", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Taints(taints).Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), + }, + candidateNodes: []string{"node-4"}, // node-3 is filtered out by TaintToleration plugin + enableNodeInclustionPolicy: true, + }, + { + name: "NodeTaintsPolicy ignored, pods spread across zone as 2/2", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Taints(taints).Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), + }, + candidateNodes: []string{"node-1", "node-2", "node-4"}, // node-3 is filtered out by TaintToleration plugin + enableNodeInclustionPolicy: true, + }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as 2/1 + // 2. to fulfil "node" constraint, pods spread across zones as 1/1/~0~/1 + // intersection of (1) and (2) returns node-4 as node-3 is filtered out by NodeAffinity plugin. + name: "two node inclusion Constraints, zone: honor/ignore, node: honor/ignore", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), + }, + candidateNodes: []string{"node-4"}, + enableNodeInclustionPolicy: true, + }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as 2/1 + // 2. to fulfil "node" constraint, pods spread across zones as 1/1/~0~/1 + // intersection of (1) and (2) returns node-4 as node-3 is filtered out by NodeAffinity plugin + name: "feature gate disabled, two node inclusion Constraints, zone: honor/ignore, node: honor/ignore", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), + }, + candidateNodes: []string{"node-4"}, + enableNodeInclustionPolicy: false, + }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as 2/2 + // 2. to fulfil "node" constraint, pods spread across zones as 1/~0~/~0~/1 + // intersection of (1) and (2) returns node-1 and node-4 as node-2, node-3 are filtered out by plugins + name: "two node inclusion Constraints, zone: ignore/ignore, node: honor/honor", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(), + }, + candidateNodes: []string{"node-1", "node-4"}, + enableNodeInclustionPolicy: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MinDomainsInPodTopologySpread, tt.enableMinDomains)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeInclusionPolicyInPodTopologySpread, tt.enableNodeInclustionPolicy)() + testCtx := initTest(t, "pts-predicate") cs := testCtx.ClientSet ns := testCtx.NS.Name defer testutils.CleanupTest(t, testCtx) - for i := 0; i < 4; i++ { - // Create nodes with labels "zone: zone-{0,1}" and "node: " to each node. - nodeName := fmt.Sprintf("node-%d", i) - zone := fmt.Sprintf("zone-%d", i/2) - _, err := createNode(cs, st.MakeNode().Name(nodeName).Label("node", nodeName).Label("zone", zone).Obj()) - if err != nil { + for i := range tt.nodes { + if _, err := createNode(cs, tt.nodes[i]); err != nil { t.Fatalf("Cannot create node: %v", err) } } diff --git a/test/integration/scheduler/scoring/priorities_test.go b/test/integration/scheduler/scoring/priorities_test.go index 52e5580b214..ec866e22b66 100644 --- a/test/integration/scheduler/scoring/priorities_test.go +++ b/test/integration/scheduler/scoring/priorities_test.go @@ -29,7 +29,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kube-scheduler/config/v1beta3" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler" configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality" @@ -57,8 +60,11 @@ var ( ) var ( - hardSpread = v1.DoNotSchedule - softSpread = v1.ScheduleAnyway + hardSpread = v1.DoNotSchedule + softSpread = v1.ScheduleAnyway + ignorePolicy = v1.NodeInclusionPolicyIgnore + honorPolicy = v1.NodeInclusionPolicyHonor + taints = []v1.Taint{{Key: v1.TaintNodeUnschedulable, Value: "", Effect: v1.TaintEffectPreferNoSchedule}} ) const ( @@ -418,83 +424,142 @@ func makeContainersWithImages(images []string) []v1.Container { // TestPodTopologySpreadScoring verifies that the PodTopologySpread Score plugin works. func TestPodTopologySpreadScoring(t *testing.T) { - testCtx := initTestSchedulerForPriorityTest(t, podtopologyspread.Name) - defer testutils.CleanupTest(t, testCtx) - cs := testCtx.ClientSet - ns := testCtx.NS.Name - - var nodes []*v1.Node - for i := 0; i < 4; i++ { - // Create nodes with labels "zone: zone-{0,1}" and "node: " to each node. - nodeName := fmt.Sprintf("node-%d", i) - zone := fmt.Sprintf("zone-%d", i/2) - node, err := createNode(cs, st.MakeNode().Name(nodeName).Label("node", nodeName).Label("zone", zone).Obj()) - if err != nil { - t.Fatalf("Cannot create node: %v", err) - } - nodes = append(nodes, node) - } - - // Taint the 0th node + pause := imageutils.GetPauseImageName() taint := v1.Taint{ Key: "k1", Value: "v1", Effect: v1.TaintEffectNoSchedule, } - if err := testutils.AddTaintToNode(cs, nodes[0].Name, taint); err != nil { - t.Fatalf("Adding taint to node failed: %v", err) - } - if err := testutils.WaitForNodeTaints(cs, nodes[0], []v1.Taint{taint}); err != nil { - t.Fatalf("Taint not seen on node: %v", err) + + // default nodes with labels "zone: zone-{0,1}" and "node: ". + defaultNodes := []*v1.Node{ + st.MakeNode().Name("node-0").Label("node", "node-0").Label("zone", "zone-0").Taints([]v1.Taint{taint}).Obj(), + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-0").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-1").Obj(), } - pause := imageutils.GetPauseImageName() tests := []struct { - name string - incomingPod *v1.Pod - existingPods []*v1.Pod - fits bool - want []string // nodes expected to schedule onto + name string + incomingPod *v1.Pod + existingPods []*v1.Pod + fits bool + nodes []*v1.Node + want []string // nodes expected to schedule onto + enableNodeInclustionPolicy bool }{ // note: naming starts at index 0 // the symbol ~X~ means that node is infeasible { name: "place pod on a ~0~/1/2/3 cluster with MaxSkew=1, node-1 is the preferred fit", - incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). - SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), existingPods: []*v1.Pod{ - st.MakePod().Namespace(ns).Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3c").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3c").Node("node-3").Label("foo", "").Container(pause).Obj(), }, - fits: true, - want: []string{"node-1"}, + fits: true, + nodes: defaultNodes, + want: []string{"node-1"}, }, { name: "combined with hardSpread constraint on a ~4~/0/1/2 cluster", - incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). - SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). - SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). Obj(), existingPods: []*v1.Pod{ - st.MakePod().Namespace(ns).Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p0b").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p0c").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p0d").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p2").Node("node-2").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0b").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0c").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0d").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: defaultNodes, + want: []string{"node-2"}, + }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as ~3~/0 + // 2. to fulfil "node" constraint, pods spread across zones as 1/~2~/0/~0~ + // node-2 and node 4 are filtered out by plugins + name: "soft constraint with two node inclusion Constraints, zone: honor/ignore, node: honor/ignore", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), }, fits: true, - want: []string{"node-2"}, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Label("foo", "").Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Obj(), + }, + want: []string{"node-3"}, + enableNodeInclustionPolicy: true, + }, + { + // 1. to fulfil "zone" constraint, pods spread across zones as ~3~/~1~ + // 2. to fulfil "node" constraint, pods spread across zones as 1/~0~/0/~0~ + // node-2 and node 4 are filtered out by plugins + name: "soft constraint with two node inclusion Constraints, zone: ignore/ignore, node: honor/honor", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeSelector(map[string]string{"foo": ""}). + SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil). + SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + nodes: []*v1.Node{ + st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(), + st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(), + st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Label("foo", "").Obj(), + st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Obj(), + }, + want: []string{"node-3"}, + enableNodeInclustionPolicy: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeInclusionPolicyInPodTopologySpread, tt.enableNodeInclustionPolicy)() + + testCtx := initTestSchedulerForPriorityTest(t, podtopologyspread.Name) + defer testutils.CleanupTest(t, testCtx) + cs := testCtx.ClientSet + ns := testCtx.NS.Name + + for i := range tt.nodes { + if _, err := createNode(cs, tt.nodes[i]); err != nil { + t.Fatalf("Cannot create node: %v", err) + } + } + + // set namespace to pods + for i := range tt.existingPods { + tt.existingPods[i].SetNamespace(ns) + } + tt.incomingPod.SetNamespace(ns) + allPods := append(tt.existingPods, tt.incomingPod) defer testutils.CleanupPods(cs, t, allPods) for _, pod := range tt.existingPods { @@ -507,6 +572,7 @@ func TestPodTopologySpreadScoring(t *testing.T) { t.Errorf("Test Failed: error while waiting for pod during test: %v", err) } } + testPod, err := cs.CoreV1().Pods(tt.incomingPod.Namespace).Create(context.TODO(), tt.incomingPod, metav1.CreateOptions{}) if err != nil && !apierrors.IsInvalid(err) { t.Fatalf("Test Failed: error while creating pod during test: %v", err)