From 6141aa53f9facb24fbc32851c59f98285d03848b Mon Sep 17 00:00:00 2001 From: sanposhiho <44139130+sanposhiho@users.noreply.github.com> Date: Sat, 26 Feb 2022 14:55:14 +0900 Subject: [PATCH] Implement MinDomains --- .../default_preemption_test.go | 9 +- .../plugins/podtopologyspread/filtering.go | 52 ++- .../podtopologyspread/filtering_test.go | 320 ++++++++++++++---- .../plugins/podtopologyspread/scoring_test.go | 87 +++-- pkg/scheduler/testing/wrappers.go | 3 +- test/integration/scheduler/predicates_test.go | 215 ++++++++---- test/integration/scheduler/priorities_test.go | 8 +- 7 files changed, 515 insertions(+), 179 deletions(-) diff --git a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go index 8fddcf4b846..9ed53732af9 100644 --- a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go +++ b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go @@ -641,8 +641,8 @@ func TestDryRunPreemption(t *testing.T) { nodeNames: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"}, testPods: []*v1.Pod{ st.MakePod().Name("p").UID("p").Label("foo", "").Priority(highPriority). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), }, initPods: []*v1.Pod{ @@ -670,7 +670,6 @@ func TestDryRunPreemption(t *testing.T) { }, expectedNumFilterCalled: []int32{5}, // node-a (3), node-b (2), node-x (0) }, - { name: "get Unschedulable in the preemption phase when the filter plugins filtering the nodes", registerPlugins: []st.RegisterPluginFunc{ @@ -1484,8 +1483,8 @@ func TestPreempt(t *testing.T) { { name: "preemption for topology spread constraints", pod: st.MakePod().Name("p").UID("p").Namespace(v1.NamespaceDefault).Label("foo", "").Priority(highPriority). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), pods: []*v1.Pod{ st.MakePod().Name("p-a1").UID("p-a1").Namespace(v1.NamespaceDefault).Node("node-a").Label("foo", "").Priority(highPriority).Obj(), diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go index 703d67d311f..cc37bcf35c6 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/filtering.go @@ -44,10 +44,38 @@ type preFilterState struct { // criticalPaths[1].MatchNum is always greater or equal to criticalPaths[0].MatchNum, but // it's not guaranteed to be the 2nd minimum match number. TpKeyToCriticalPaths map[string]*criticalPaths + // TpKeyToDomainsNum is keyed with topologyKey, and valued with the number of domains. + TpKeyToDomainsNum map[string]int // TpPairToMatchNum is keyed with topologyPair, and valued with the number of matching pods. TpPairToMatchNum map[topologyPair]int } +// minMatchNum returns the global minimum for the calculation of skew while taking MinDomains into account. +func (s *preFilterState) minMatchNum(tpKey string, minDomains int32, enableMinDomainsInPodTopologySpread bool) (int, error) { + paths, ok := s.TpKeyToCriticalPaths[tpKey] + if !ok { + return 0, fmt.Errorf("failed to retrieve path by topology key") + } + + minMatchNum := paths[0].MatchNum + if !enableMinDomainsInPodTopologySpread { + return minMatchNum, nil + } + + domainsNum, ok := s.TpKeyToDomainsNum[tpKey] + if !ok { + return 0, fmt.Errorf("failed to retrieve the number of domains by topology key") + } + + if domainsNum < int(minDomains) { + // When the number of eligible domains with matching topology keys is less than `minDomains`, + // it treats "global minimum" as 0. + minMatchNum = 0 + } + + return minMatchNum, nil +} + // Clone makes a copy of the given state. func (s *preFilterState) Clone() framework.StateData { if s == nil { @@ -57,7 +85,9 @@ func (s *preFilterState) Clone() framework.StateData { // Constraints are shared because they don't change. Constraints: s.Constraints, TpKeyToCriticalPaths: make(map[string]*criticalPaths, len(s.TpKeyToCriticalPaths)), - TpPairToMatchNum: make(map[topologyPair]int, len(s.TpPairToMatchNum)), + // The number of domains does not change as a result of AddPod/RemovePod methods on PreFilter Extensions + TpKeyToDomainsNum: s.TpKeyToDomainsNum, + TpPairToMatchNum: make(map[topologyPair]int, len(s.TpPairToMatchNum)), } for tpKey, paths := range s.TpKeyToCriticalPaths { copy.TpKeyToCriticalPaths[tpKey] = &criticalPaths{paths[0], paths[1]} @@ -257,6 +287,12 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod) s.TpPairToMatchNum[tp] += count } } + if pl.enableMinDomainsInPodTopologySpread { + s.TpKeyToDomainsNum = make(map[string]int, len(constraints)) + for tp := range s.TpPairToMatchNum { + s.TpKeyToDomainsNum[tp.key]++ + } + } // calculate min match for each topology pair for i := 0; i < len(constraints); i++ { @@ -302,15 +338,15 @@ func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.C } pair := topologyPair{key: tpKey, value: tpVal} - paths, ok := s.TpKeyToCriticalPaths[tpKey] - if !ok { - // error which should not happen - klog.ErrorS(nil, "Internal error occurred while retrieving paths from topology key", "topologyKey", tpKey, "paths", s.TpKeyToCriticalPaths) + + // judging criteria: + // 'existing matching num' + 'if self-match (1 or 0)' - 'global minimum' <= 'maxSkew' + minMatchNum, err := s.minMatchNum(tpKey, c.MinDomains, pl.enableMinDomainsInPodTopologySpread) + if err != nil { + klog.ErrorS(err, "Internal error occurred while retrieving value precalculated in PreFilter", "topologyKey", tpKey, "paths", s.TpKeyToCriticalPaths) continue } - // judging criteria: - // 'existing matching num' + 'if self-match (1 or 0)' - 'global min matching num' <= 'maxSkew' - minMatchNum := paths[0].MatchNum + matchNum := 0 if tpCount, ok := s.TpPairToMatchNum[pair]; ok { matchNum = tpCount diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go b/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go index c6ebe983d80..f51aa0a0367 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go @@ -34,6 +34,7 @@ import ( frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" "k8s.io/kubernetes/pkg/scheduler/internal/cache" st "k8s.io/kubernetes/pkg/scheduler/testing" + "k8s.io/utils/pointer" ) var cmpOpts = []cmp.Option{ @@ -64,6 +65,7 @@ func TestPreFilterState(t *testing.T) { pod *v1.Pod nodes []*v1.Node existingPods []*v1.Pod + enableMinDomains bool objs []runtime.Object defaultConstraints []v1.TopologySpreadConstraint want *preFilterState @@ -71,7 +73,7 @@ func TestPreFilterState(t *testing.T) { { name: "clean cluster with one spreadConstraint", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 5, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj(), + 5, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -100,7 +102,7 @@ func TestPreFilterState(t *testing.T) { { name: "normal case with one spreadConstraint", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, fooSelector, + 1, "zone", v1.DoNotSchedule, fooSelector, nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -136,7 +138,7 @@ func TestPreFilterState(t *testing.T) { { name: "normal case with one spreadConstraint, on a 3-zone cluster", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -175,7 +177,7 @@ func TestPreFilterState(t *testing.T) { { name: "namespace mismatch doesn't count", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, fooSelector, + 1, "zone", v1.DoNotSchedule, fooSelector, nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -211,8 +213,8 @@ func TestPreFilterState(t *testing.T) { { name: "normal case with two spreadConstraints", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector). - SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -261,10 +263,10 @@ func TestPreFilterState(t *testing.T) { { name: "soft spreadConstraints should be bypassed", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector). - SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector). - SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -311,8 +313,8 @@ func TestPreFilterState(t *testing.T) { { name: "different labelSelectors - simple version", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector). - SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -354,8 +356,8 @@ func TestPreFilterState(t *testing.T) { { name: "different labelSelectors - complex pods", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector). - SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, barSelector, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -403,8 +405,8 @@ func TestPreFilterState(t *testing.T) { name: "two spreadConstraints, and with podAffinity", pod: st.MakePod().Name("p").Label("foo", ""). NodeAffinityNotIn("node", []string{"node-x"}). // exclude node-x - SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector). - SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -496,8 +498,8 @@ func TestPreFilterState(t *testing.T) { { name: "default constraints and a service, but pod has constraints", pod: st.MakePod().Name("p").Label("foo", "bar").Label("baz", "tar"). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("baz", "tar").Obj()). - SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("fot", "rok").Obj()).Obj(), + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("baz", "tar").Obj(), nil). + SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("fot", "rok").Obj(), nil).Obj(), defaultConstraints: []v1.TopologySpreadConstraint{ {MaxSkew: 2, TopologyKey: "node", WhenUnsatisfiable: v1.DoNotSchedule}, }, @@ -530,6 +532,61 @@ func TestPreFilterState(t *testing.T) { }, want: &preFilterState{}, }, + { + name: "TpKeyToDomainsNum is calculated when MinDomains is enabled", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(1, "zone", v1.DoNotSchedule, fooSelector, nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, fooSelector, nil). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(), + st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(), + st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(), + }, + enableMinDomains: true, + want: &preFilterState{ + Constraints: []topologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "zone", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + }, + { + MaxSkew: 1, + TopologyKey: "node", + Selector: mustConvertLabelSelectorAsSelector(t, fooSelector), + MinDomains: 1, + }, + }, + TpKeyToCriticalPaths: map[string]*criticalPaths{ + "zone": {{"zone1", 3}, {"zone2", 4}}, + "node": {{"node-x", 0}, {"node-b", 1}}, + }, + TpPairToMatchNum: map[topologyPair]int{ + {key: "zone", value: "zone1"}: 3, + {key: "zone", value: "zone2"}: 4, + {key: "node", value: "node-a"}: 2, + {key: "node", value: "node-b"}: 1, + {key: "node", value: "node-x"}: 0, + {key: "node", value: "node-y"}: 4, + }, + TpKeyToDomainsNum: map[string]int{ + "zone": 2, + "node": 4, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -539,7 +596,10 @@ func TestPreFilterState(t *testing.T) { DefaultConstraints: tt.defaultConstraints, DefaultingType: config.ListDefaulting, } + p := plugintesting.SetupPluginWithInformers(ctx, t, topologySpreadFunc, args, cache.NewSnapshot(tt.existingPods, tt.nodes), tt.objs) + p.(*PodTopologySpread).enableMinDomainsInPodTopologySpread = tt.enableMinDomains + cs := framework.NewCycleState() if s := p.(*PodTopologySpread).PreFilter(ctx, cs, tt.pod); !s.IsSuccess() { t.Fatal(s.AsError()) @@ -576,7 +636,7 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "node a and b both impact current min match", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: nil, // it's an empty cluster @@ -599,7 +659,7 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "only node a impacts current min match", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -624,7 +684,7 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "add a pod in a different namespace doesn't change topologyKeyToMinPodsMap", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-a1").Namespace("ns1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -649,7 +709,7 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "add pod on non-critical node won't trigger re-calculation", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -674,8 +734,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "node a and x both impact topologyKeyToMinPodsMap on zone and node", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: nil, // it's an empty cluster @@ -701,8 +761,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "only node a impacts topologyKeyToMinPodsMap on zone and node", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -730,8 +790,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "node a impacts topologyKeyToMinPodsMap on node, node x impacts topologyKeyToMinPodsMap on zone", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -763,8 +823,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "Constraints hold different labelSelectors, node a impacts topologyKeyToMinPodsMap on zone", preemptor: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), existingPods: []*v1.Pod{ @@ -804,8 +864,8 @@ func TestPreFilterStateAddPod(t *testing.T) { { name: "Constraints hold different labelSelectors, node a impacts topologyKeyToMinPodsMap on both zone and node", preemptor: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), addedPod: st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Label("bar", "").Obj(), existingPods: []*v1.Pod{ @@ -895,7 +955,7 @@ func TestPreFilterStateRemovePod(t *testing.T) { // So preemption is triggered. name: "one spreadConstraint on zone, topologyKeyToMinPodsMap unchanged", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -923,7 +983,7 @@ func TestPreFilterStateRemovePod(t *testing.T) { { name: "one spreadConstraint on node, topologyKeyToMinPodsMap changed", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -953,7 +1013,7 @@ func TestPreFilterStateRemovePod(t *testing.T) { { name: "delete an irrelevant pod won't help", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -984,7 +1044,7 @@ func TestPreFilterStateRemovePod(t *testing.T) { { name: "delete a non-existing pod won't help", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1015,8 +1075,8 @@ func TestPreFilterStateRemovePod(t *testing.T) { { name: "two spreadConstraints", preemptor: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1095,7 +1155,7 @@ func BenchmarkFilter(b *testing.B) { { name: "1000nodes/single-constraint-zone", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelTopologyZone, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelTopologyZone, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -1104,7 +1164,7 @@ func BenchmarkFilter(b *testing.B) { { name: "1000nodes/single-constraint-node", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -1113,8 +1173,8 @@ func BenchmarkFilter(b *testing.B) { { name: "1000nodes/two-Constraints-zone-node", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, v1.LabelTopologyZone, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, v1.LabelHostname, v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, v1.LabelTopologyZone, v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -1161,16 +1221,17 @@ func mustConvertLabelSelectorAsSelector(t *testing.T, ls *metav1.LabelSelector) func TestSingleConstraint(t *testing.T) { tests := []struct { - name string - pod *v1.Pod - nodes []*v1.Node - existingPods []*v1.Pod - wantStatusCode map[string]framework.Code + name string + pod *v1.Pod + nodes []*v1.Node + existingPods []*v1.Pod + enableMinDomains bool + wantStatusCode map[string]framework.Code }{ { name: "no existing pods", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1188,7 +1249,7 @@ func TestSingleConstraint(t *testing.T) { { name: "no existing pods, incoming pod doesn't match itself", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), + 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1206,7 +1267,7 @@ func TestSingleConstraint(t *testing.T) { { name: "existing pods in a different namespace do not count", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1230,7 +1291,7 @@ func TestSingleConstraint(t *testing.T) { { name: "pods spread across zones as 3/3, all nodes fit", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1258,7 +1319,7 @@ func TestSingleConstraint(t *testing.T) { // can cause unexpected behavior name: "pods spread across zones as 1/2 due to absence of label 'zone' on node-b", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1282,7 +1343,7 @@ func TestSingleConstraint(t *testing.T) { { name: "pod cannot be scheduled as all nodes don't have label 'rack'", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "rack", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "rack", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1296,7 +1357,7 @@ func TestSingleConstraint(t *testing.T) { { name: "pods spread across nodes as 2/1/0/3, only node-x fits", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1322,7 +1383,7 @@ func TestSingleConstraint(t *testing.T) { { name: "pods spread across nodes as 2/1/0/3, maxSkew is 2, node-b and node-x fit", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 2, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 2, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1352,7 +1413,7 @@ func TestSingleConstraint(t *testing.T) { // as the incoming pod doesn't have label "foo" name: "pods spread across nodes as 2/1/0/3, but pod doesn't match itself", pod: st.MakePod().Name("p").Label("bar", "").SpreadConstraint( - 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1384,7 +1445,7 @@ func TestSingleConstraint(t *testing.T) { name: "incoming pod has nodeAffinity, pods spread as 2/~1~/~0~/3, hence node-a fits", pod: st.MakePod().Name("p").Label("foo", ""). NodeAffinityIn("node", []string{"node-a", "node-y"}). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1410,7 +1471,7 @@ func TestSingleConstraint(t *testing.T) { { name: "terminating Pods should be excluded", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), + 1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), @@ -1430,7 +1491,7 @@ func TestSingleConstraint(t *testing.T) { name: "incoming pod has nodeAffinity, pods spread as 0/~2~/0/1, hence node-a fits", pod: st.MakePod().Name("p").Label("foo", ""). NodeAffinityNotIn("node", []string{"node-b"}). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1450,12 +1511,125 @@ func TestSingleConstraint(t *testing.T) { "node-y": framework.Unschedulable, }, }, + { + name: "pods spread across nodes as 2/2/1, maxSkew is 2, and the number of domains < minDomains, then the third node fits", + pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( + 2, + "node", + v1.DoNotSchedule, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(4), // larger than the number of domains(3) + ).Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + enableMinDomains: true, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-c": framework.Success, + }, + }, + { + name: "pods spread across nodes as 2/2/1, maxSkew is 2, and the number of domains > minDomains, then the all nodes fit", + pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( + 2, + "node", + v1.DoNotSchedule, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(2), // smaller than the number of domains(3) + ).Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + enableMinDomains: true, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Success, + "node-b": framework.Success, + "node-c": framework.Success, + }, + }, + { + name: "pods spread across zone as 2/1, maxSkew is 2 and the number of domains < minDomains, then the third and fourth nodes fit", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(2, + "zone", + v1.DoNotSchedule, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(3), // larger than the number of domains(2) + ).Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + enableMinDomains: true, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Unschedulable, + "node-b": framework.Unschedulable, + "node-x": framework.Success, + "node-y": framework.Success, + }, + }, + { + name: "pods spread across zone as 2/1, maxSkew is 2 and the number of domains > minDomains, then the all nodes fit", + pod: st.MakePod().Name("p").Label("foo", ""). + SpreadConstraint(2, + "zone", + v1.DoNotSchedule, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(1), // smaller than the number of domains(2) + ).Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(), + st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-b1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(), + }, + enableMinDomains: true, + wantStatusCode: map[string]framework.Code{ + "node-a": framework.Success, + "node-b": framework.Success, + "node-x": framework.Success, + "node-y": framework.Success, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { snapshot := cache.NewSnapshot(tt.existingPods, tt.nodes) pl := plugintesting.SetupPlugin(t, topologySpreadFunc, &config.PodTopologySpreadArgs{DefaultingType: config.ListDefaulting}, snapshot) p := pl.(*PodTopologySpread) + p.enableMinDomainsInPodTopologySpread = tt.enableMinDomains state := framework.NewCycleState() preFilterStatus := p.PreFilter(context.Background(), state, tt.pod) if !preFilterStatus.IsSuccess() { @@ -1487,8 +1661,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-x name: "two Constraints on zone and node, spreads = [3/3, 2/1/0/3]", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1517,8 +1691,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns no node name: "two Constraints on zone and node, spreads = [3/4, 2/1/0/4]", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1548,8 +1722,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-x name: "Constraints hold different labelSelectors, spreads = [1/0, 1/0/0/1]", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1574,8 +1748,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns no node name: "Constraints hold different labelSelectors, spreads = [1/0, 0/0/1/1]", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1601,8 +1775,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-b name: "Constraints hold different labelSelectors, spreads = [2/3, 1/0/0/1]", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1630,8 +1804,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-a and node-b name: "Constraints hold different labelSelectors but pod doesn't match itself on 'zone' constraint", pod: st.MakePod().Name("p").Label("bar", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), @@ -1657,8 +1831,8 @@ func TestMultipleConstraints(t *testing.T) { // intersection of (1) and (2) returns node-b name: "two Constraints on zone and node, absence of label 'node' on node-x, spreads = [1/1, 1/0/0/1]", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(), diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go b/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go index 4d6d645b88f..b08a7d1446f 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go @@ -52,8 +52,8 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { { name: "normal case", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), @@ -89,8 +89,8 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { { name: "node-x doesn't have label zone", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(), @@ -223,8 +223,8 @@ func TestPreScoreStateEmptyNodes(t *testing.T) { name: "default constraints and a replicaset, but pod has constraints", pod: st.MakePod().Name("p").Namespace("default").Label("foo", "bar").Label("baz", "sup"). OwnerReference("rs1", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")). - SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj()). - SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("baz", "sup").Obj()).Obj(), + SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj(), nil). + SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("baz", "sup").Obj(), nil).Obj(), config: config.PodTopologySpreadArgs{ DefaultConstraints: []v1.TopologySpreadConstraint{ {MaxSkew: 2, TopologyKey: "galaxy", WhenUnsatisfiable: v1.ScheduleAnyway}, @@ -308,7 +308,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "one constraint on node, no existing pods", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), @@ -323,7 +323,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // if there is only one candidate node, it should be scored to 100 name: "one constraint on node, only one node is candidate", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -343,7 +343,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "one constraint on node, all nodes have the same number of matching pods", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -362,7 +362,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 2/1/0/3. name: "one constraint on node, all 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -389,7 +389,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "one constraint on node, all 4 nodes are candidates, maxSkew=2", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(2, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(2, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), // matching pods spread as 2/1/0/3. existingPods: []*v1.Pod{ @@ -417,7 +417,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "one constraint on node, all 4 nodes are candidates, maxSkew=3", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(3, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(3, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ // matching pods spread as 4/3/2/1. @@ -484,7 +484,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 4/2/1/~3~ (node4 is not a candidate) name: "one constraint on node, 3 out of 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -516,7 +516,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 4/?2?/1/~3~, total = 4+?+1 = 5 (as node2 is problematic) name: "one constraint on node, 3 out of 4 nodes are candidates, one node doesn't match topology key", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -548,7 +548,7 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 4/2/1/~3~ name: "one constraint on zone, 3 out of 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -580,8 +580,8 @@ func TestPodTopologySpreadScore(t *testing.T) { // matching pods spread as 2/~1~/2/~4~. name: "two Constraints on zone and node, 2 out of 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -620,8 +620,8 @@ func TestPodTopologySpreadScore(t *testing.T) { // For the second constraint (node): the matching pods spread as 0/1/0/1 name: "two Constraints on zone and node, with different labelSelectors", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -648,8 +648,8 @@ func TestPodTopologySpreadScore(t *testing.T) { // For the second constraint (node): the matching pods spread as 0/1/0/1 name: "two Constraints on zone and node, with different labelSelectors, some nodes have 0 pods", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-b1").Node("node-b").Label("bar", "").Obj(), @@ -675,8 +675,8 @@ func TestPodTopologySpreadScore(t *testing.T) { // For the second constraint (node): the matching pods spread as 0/1/0/~1~ name: "two Constraints on zone and node, with different labelSelectors, 3 out of 4 nodes are candidates", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), @@ -701,7 +701,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "existing pods in a different namespace do not count", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Name("p-a1").Namespace("ns1").Node("node-a").Label("foo", "").Obj(), @@ -721,7 +721,7 @@ func TestPodTopologySpreadScore(t *testing.T) { { name: "terminating Pods should be excluded", pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( - 1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), + 1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil, ).Obj(), nodes: []*v1.Node{ st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(), @@ -736,6 +736,35 @@ func TestPodTopologySpreadScore(t *testing.T) { {Name: "node-b", Score: 0}, }, }, + { + // This test is artificial. In the real world, API Server would fail a pod's creation + // when non-default minDomains is specified along with SchedulingAnyway. + name: "minDomains has no effect when ScheduleAnyway", + pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( + 2, + "node", + v1.ScheduleAnyway, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(10), // larger than the number of domains(3) + ).Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node-a").Label("node", "node-a").Obj(), + st.MakeNode().Name("node-b").Label("node", "node-b").Obj(), + st.MakeNode().Name("node-c").Label("node", "node-c").Obj(), + }, + existingPods: []*v1.Pod{ + st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(), + st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-b2").Node("node-b").Label("foo", "").Obj(), + st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(), + }, + want: []framework.NodeScore{ + {Name: "node-a", Score: 75}, + {Name: "node-b", Score: 75}, + {Name: "node-c", Score: 100}, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -784,7 +813,7 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) { { name: "1000nodes/single-constraint-zone", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -793,7 +822,7 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) { { name: "1000nodes/single-constraint-node", pod: st.MakePod().Name("p").Label("foo", ""). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, @@ -802,8 +831,8 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) { { name: "1000nodes/two-Constraints-zone-node", pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). - SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj()). + SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil). Obj(), existingPodsNum: 10000, allNodesNum: 1000, diff --git a/pkg/scheduler/testing/wrappers.go b/pkg/scheduler/testing/wrappers.go index b42f2a1d509..8aec518c506 100644 --- a/pkg/scheduler/testing/wrappers.go +++ b/pkg/scheduler/testing/wrappers.go @@ -396,12 +396,13 @@ func (p *PodWrapper) PodAntiAffinityExists(labelKey, topologyKey string, kind Po // SpreadConstraint constructs a TopologySpreadConstraint object and injects // into the inner pod. -func (p *PodWrapper) SpreadConstraint(maxSkew int, tpKey string, mode v1.UnsatisfiableConstraintAction, selector *metav1.LabelSelector) *PodWrapper { +func (p *PodWrapper) SpreadConstraint(maxSkew int, tpKey string, mode v1.UnsatisfiableConstraintAction, selector *metav1.LabelSelector, minDomains *int32) *PodWrapper { c := v1.TopologySpreadConstraint{ MaxSkew: int32(maxSkew), TopologyKey: tpKey, WhenUnsatisfiable: mode, LabelSelector: selector, + MinDomains: minDomains, } p.Spec.TopologySpreadConstraints = append(p.Spec.TopologySpreadConstraints, c) return p diff --git a/test/integration/scheduler/predicates_test.go b/test/integration/scheduler/predicates_test.go index 8570eb3695c..9716406ef29 100644 --- a/test/integration/scheduler/predicates_test.go +++ b/test/integration/scheduler/predicates_test.go @@ -26,10 +26,14 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" st "k8s.io/kubernetes/pkg/scheduler/testing" testutils "k8s.io/kubernetes/test/integration/util" imageutils "k8s.io/kubernetes/test/utils/image" + "k8s.io/utils/pointer" ) // This file tests the scheduler predicates functionality. @@ -1032,124 +1036,217 @@ func TestInterPodAffinityWithNamespaceSelector(t *testing.T) { } } -// TestEvenPodsSpreadPredicate verifies that EvenPodsSpread predicate functions well. -func TestEvenPodsSpreadPredicate(t *testing.T) { - testCtx := initTest(t, "eps-predicate") - cs := testCtx.ClientSet - ns := testCtx.NS.Name - defer testutils.CleanupTest(t, testCtx) - - for i := 0; i < 4; i++ { - // Create nodes with labels "zone: zone-{0,1}" and "node: " to each node. - nodeName := fmt.Sprintf("node-%d", i) - zone := fmt.Sprintf("zone-%d", i/2) - _, err := createNode(cs, st.MakeNode().Name(nodeName).Label("node", nodeName).Label("zone", zone).Obj()) - if err != nil { - t.Fatalf("Cannot create node: %v", err) - } - } - +// TestPodTopologySpreadFilter verifies that EvenPodsSpread predicate functions well. +func TestPodTopologySpreadFilter(t *testing.T) { pause := imageutils.GetPauseImageName() tests := []struct { - name string - incomingPod *v1.Pod - existingPods []*v1.Pod - fits bool - candidateNodes []string // nodes expected to schedule onto + name string + incomingPod *v1.Pod + existingPods []*v1.Pod + fits bool + candidateNodes []string // nodes expected to schedule onto + enableMinDomains bool }{ // note: naming starts at index 0 { name: "place pod on a 1/1/0/1 cluster with MaxSkew=1, node-2 is the only fit", - incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ - st.MakePod().Namespace(ns).Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, candidateNodes: []string{"node-2"}, }, { name: "place pod on a 2/0/0/1 cluster with MaxSkew=2, node-{1,2,3} are good fits", - incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). - SpreadConstraint(2, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint(2, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ - st.MakePod().Namespace(ns).Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p0b").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0b").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, candidateNodes: []string{"node-1", "node-2", "node-3"}, }, { name: "pod is required to be placed on zone0, so only node-1 fits", - incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). NodeAffinityIn("zone", []string{"zone-0"}). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ - st.MakePod().Namespace(ns).Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, candidateNodes: []string{"node-1"}, }, { name: "two constraints: pod can only be placed to zone-1/node-2", - incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). - SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ - st.MakePod().Namespace(ns).Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, candidateNodes: []string{"node-2"}, }, { name: "pod cannot be placed onto any node", - incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster - SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ - st.MakePod().Namespace(ns).Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), - st.MakePod().Namespace(ns).Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: false, }, { name: "high priority pod can preempt others", - incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause).Priority(100). + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).Priority(100). NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster - SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ - st.MakePod().ZeroTerminationGracePeriod().Namespace(ns).Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), - st.MakePod().ZeroTerminationGracePeriod().Namespace(ns).Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), - st.MakePod().ZeroTerminationGracePeriod().Namespace(ns).Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), - st.MakePod().ZeroTerminationGracePeriod().Namespace(ns).Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), - st.MakePod().ZeroTerminationGracePeriod().Namespace(ns).Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), }, fits: true, candidateNodes: []string{"node-1", "node-2", "node-3"}, }, + { + name: "pods spread across nodes as 2/2/1, maxSkew is 2, and the number of domains < minDomains, then the third node fits", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster + SpreadConstraint( + 2, + "node", + hardSpread, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(4), // larger than the number of domains (= 3) + ). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().ZeroTerminationGracePeriod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + candidateNodes: []string{"node-3"}, + enableMinDomains: true, + }, + { + name: "pods spread across nodes as 2/2/1, maxSkew is 2, and the number of domains > minDomains, then the all nodes fit", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster + SpreadConstraint( + 2, + "node", + hardSpread, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(2), // smaller than the number of domains (= 3) + ). + Obj(), + existingPods: []*v1.Pod{ + st.MakePod().ZeroTerminationGracePeriod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p1b").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + candidateNodes: []string{"node-1", "node-2", "node-3"}, + enableMinDomains: true, + }, + { + name: "pods spread across zone as 2/1, maxSkew is 2 and the number of domains < minDomains, then the third and fourth nodes fit", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint( + 2, + "zone", + v1.DoNotSchedule, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(3), // larger than the number of domains(2) + ).Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-0").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-2").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + candidateNodes: []string{"node-2", "node-3"}, + enableMinDomains: true, + }, + { + name: "pods spread across zone as 2/1, maxSkew is 2 and the number of domains < minDomains, then the all nodes fit", + incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause). + SpreadConstraint( + 2, + "zone", + v1.DoNotSchedule, + st.MakeLabelSelector().Exists("foo").Obj(), + pointer.Int32(1), // smaller than the number of domains(2) + ).Obj(), + existingPods: []*v1.Pod{ + st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(), + st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(), + }, + fits: true, + candidateNodes: []string{"node-0", "node-1", "node-2", "node-3"}, + enableMinDomains: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MinDomainsInPodTopologySpread, tt.enableMinDomains)() + testCtx := initTest(t, "pts-predicate") + cs := testCtx.ClientSet + ns := testCtx.NS.Name + defer testutils.CleanupTest(t, testCtx) + + for i := 0; i < 4; i++ { + // Create nodes with labels "zone: zone-{0,1}" and "node: " to each node. + nodeName := fmt.Sprintf("node-%d", i) + zone := fmt.Sprintf("zone-%d", i/2) + _, err := createNode(cs, st.MakeNode().Name(nodeName).Label("node", nodeName).Label("zone", zone).Obj()) + if err != nil { + t.Fatalf("Cannot create node: %v", err) + } + } + + // set namespace to pods + for i := range tt.existingPods { + tt.existingPods[i].SetNamespace(ns) + } + tt.incomingPod.SetNamespace(ns) + allPods := append(tt.existingPods, tt.incomingPod) defer testutils.CleanupPods(cs, t, allPods) + for _, pod := range tt.existingPods { createdPod, err := cs.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) if err != nil { diff --git a/test/integration/scheduler/priorities_test.go b/test/integration/scheduler/priorities_test.go index 2289676736a..dca57d17e8f 100644 --- a/test/integration/scheduler/priorities_test.go +++ b/test/integration/scheduler/priorities_test.go @@ -22,7 +22,7 @@ import ( "strings" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -441,7 +441,7 @@ func TestPodTopologySpreadScoring(t *testing.T) { { name: "place pod on a ~0~/1/2/3 cluster with MaxSkew=1, node-1 is the preferred fit", incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). - SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Namespace(ns).Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(), @@ -457,8 +457,8 @@ func TestPodTopologySpreadScoring(t *testing.T) { { name: "combined with hardSpread constraint on a ~4~/0/1/2 cluster", incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause). - SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj()). - SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). + SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). + SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil). Obj(), existingPods: []*v1.Pod{ st.MakePod().Namespace(ns).Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(),