From 4b31b554992fc50a65fdeca47bbd1f48f69a91b3 Mon Sep 17 00:00:00 2001 From: Aldo Culquicondor Date: Wed, 25 Mar 2020 13:29:27 -0400 Subject: [PATCH 1/2] Count spreading node matches for hostname topology in Score --- .../plugins/podtopologyspread/common.go | 14 +++++++ .../plugins/podtopologyspread/scoring.go | 37 +++++++++---------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/common.go b/pkg/scheduler/framework/plugins/podtopologyspread/common.go index b87af00c88e..fa53a9ac48b 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/common.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/common.go @@ -82,3 +82,17 @@ func filterTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint, } return result, nil } + +func countPodsMatchSelector(pods []*v1.Pod, selector labels.Selector, ns string) int { + count := 0 + for _, p := range pods { + // Bypass terminating Pod (see #87621). + if p.DeletionTimestamp != nil || p.Namespace != ns { + continue + } + if selector.Matches(labels.Set(p.Labels)) { + count++ + } + } + return count +} diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go b/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go index 6686361f85a..bac11cea11a 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/scoring.go @@ -23,7 +23,6 @@ import ( "sync/atomic" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" pluginhelper "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" @@ -74,6 +73,10 @@ func (pl *PodTopologySpread) initPreScoreState(s *preScoreState, pod *v1.Pod, fi continue } for _, constraint := range s.Constraints { + // per-node counts are calculated during Score. + if constraint.TopologyKey == v1.LabelHostname { + continue + } pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]} if s.TopologyPairToPodCounts[pair] == nil { s.TopologyPairToPodCounts[pair] = new(int64) @@ -104,7 +107,7 @@ func (pl *PodTopologySpread) PreScore( } state := &preScoreState{ - NodeNameSet: sets.String{}, + NodeNameSet: make(sets.String, len(filteredNodes)), TopologyPairToPodCounts: make(map[topologyPair]*int64), } err = pl.initPreScoreState(state, pod, filteredNodes) @@ -135,22 +138,13 @@ func (pl *PodTopologySpread) PreScore( pair := topologyPair{key: c.TopologyKey, value: node.Labels[c.TopologyKey]} // If current topology pair is not associated with any candidate node, // continue to avoid unnecessary calculation. - if state.TopologyPairToPodCounts[pair] == nil { + // Per-node counts are also skipped, as they are done during Score. + tpCount := state.TopologyPairToPodCounts[pair] + if tpCount == nil { continue } - - // indicates how many pods (on current node) match the . - matchSum := int64(0) - for _, existingPod := range nodeInfo.Pods() { - // Bypass terminating Pod (see #87621). - if existingPod.DeletionTimestamp != nil || existingPod.Namespace != pod.Namespace { - continue - } - if c.Selector.Matches(labels.Set(existingPod.Labels)) { - matchSum++ - } - } - atomic.AddInt64(state.TopologyPairToPodCounts[pair], matchSum) + count := countPodsMatchSelector(nodeInfo.Pods(), c.Selector, pod.Namespace) + atomic.AddInt64(tpCount, int64(count)) } } parallelize.Until(ctx, len(allNodes), processAllNode) @@ -184,9 +178,14 @@ func (pl *PodTopologySpread) Score(ctx context.Context, cycleState *framework.Cy var score int64 for _, c := range s.Constraints { if tpVal, ok := node.Labels[c.TopologyKey]; ok { - pair := topologyPair{key: c.TopologyKey, value: tpVal} - matchSum := *s.TopologyPairToPodCounts[pair] - score += matchSum + if c.TopologyKey == v1.LabelHostname { + count := countPodsMatchSelector(nodeInfo.Pods(), c.Selector, pod.Namespace) + score += int64(count) + } else { + pair := topologyPair{key: c.TopologyKey, value: tpVal} + matchSum := *s.TopologyPairToPodCounts[pair] + score += matchSum + } } } return score, nil From d2b1903149a942dfaf53a881abdefbe27699acc9 Mon Sep 17 00:00:00 2001 From: Aldo Culquicondor Date: Wed, 25 Mar 2020 15:18:05 -0400 Subject: [PATCH 2/2] Calculate scores in parallel on spreading benchmarks This is closer to what happens in the core scheduler Signed-off-by: Aldo Culquicondor --- .../plugins/defaultpodtopologyspread/BUILD | 1 + .../default_pod_topology_spread_perf_test.go | 16 ++++++++-------- .../framework/plugins/podtopologyspread/BUILD | 1 + .../plugins/podtopologyspread/scoring_test.go | 16 ++++++++-------- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pkg/scheduler/framework/plugins/defaultpodtopologyspread/BUILD b/pkg/scheduler/framework/plugins/defaultpodtopologyspread/BUILD index 3547d3f9c47..a33db5454d4 100644 --- a/pkg/scheduler/framework/plugins/defaultpodtopologyspread/BUILD +++ b/pkg/scheduler/framework/plugins/defaultpodtopologyspread/BUILD @@ -27,6 +27,7 @@ go_test( deps = [ "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/internal/cache:go_default_library", + "//pkg/scheduler/internal/parallelize:go_default_library", "//pkg/scheduler/testing:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/scheduler/framework/plugins/defaultpodtopologyspread/default_pod_topology_spread_perf_test.go b/pkg/scheduler/framework/plugins/defaultpodtopologyspread/default_pod_topology_spread_perf_test.go index 5e03d3bce9d..212e70a1f26 100644 --- a/pkg/scheduler/framework/plugins/defaultpodtopologyspread/default_pod_topology_spread_perf_test.go +++ b/pkg/scheduler/framework/plugins/defaultpodtopologyspread/default_pod_topology_spread_perf_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/kubernetes/fake" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/pkg/scheduler/internal/cache" + "k8s.io/kubernetes/pkg/scheduler/internal/parallelize" st "k8s.io/kubernetes/pkg/scheduler/testing" ) @@ -76,15 +77,14 @@ func BenchmarkTestSelectorSpreadPriority(b *testing.B) { if !status.IsSuccess() { b.Fatalf("unexpected error: %v", status) } - var gotList framework.NodeScoreList - for _, node := range filteredNodes { - score, status := plugin.Score(ctx, state, pod, node.Name) - if !status.IsSuccess() { - b.Errorf("unexpected error: %v", status) - } - gotList = append(gotList, framework.NodeScore{Name: node.Name, Score: score}) + gotList := make(framework.NodeScoreList, len(filteredNodes)) + scoreNode := func(i int) { + n := filteredNodes[i] + score, _ := plugin.Score(ctx, state, pod, n.Name) + gotList[i] = framework.NodeScore{Name: n.Name, Score: score} } - status = plugin.NormalizeScore(context.Background(), state, pod, gotList) + parallelize.Until(ctx, len(filteredNodes), scoreNode) + status = plugin.NormalizeScore(ctx, state, pod, gotList) if !status.IsSuccess() { b.Fatal(status) } diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/BUILD b/pkg/scheduler/framework/plugins/podtopologyspread/BUILD index 78a644051bc..d24ad02b7dc 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/BUILD +++ b/pkg/scheduler/framework/plugins/podtopologyspread/BUILD @@ -41,6 +41,7 @@ go_test( deps = [ "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/internal/cache:go_default_library", + "//pkg/scheduler/internal/parallelize:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/testing:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go b/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go index 60583b7f021..c34503f02b5 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/kubernetes/fake" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/pkg/scheduler/internal/cache" + "k8s.io/kubernetes/pkg/scheduler/internal/parallelize" st "k8s.io/kubernetes/pkg/scheduler/testing" "k8s.io/utils/pointer" ) @@ -746,19 +747,18 @@ func BenchmarkTestDefaultEvenPodsSpreadPriority(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - var gotList framework.NodeScoreList status := p.PreScore(ctx, state, pod, filteredNodes) if !status.IsSuccess() { b.Fatalf("unexpected error: %v", status) } - for _, n := range filteredNodes { - score, status := p.Score(context.Background(), state, pod, n.Name) - if !status.IsSuccess() { - b.Fatalf("unexpected error: %v", status) - } - gotList = append(gotList, framework.NodeScore{Name: n.Name, Score: score}) + gotList := make(framework.NodeScoreList, len(filteredNodes)) + scoreNode := func(i int) { + n := filteredNodes[i] + score, _ := p.Score(ctx, state, pod, n.Name) + gotList[i] = framework.NodeScore{Name: n.Name, Score: score} } - status = p.NormalizeScore(context.Background(), state, pod, gotList) + parallelize.Until(ctx, len(filteredNodes), scoreNode) + status = p.NormalizeScore(ctx, state, pod, gotList) if !status.IsSuccess() { b.Fatal(status) }