From cbf1ea5e6893bd4fcc41f663281471d31b218f31 Mon Sep 17 00:00:00 2001 From: sanposhiho <44139130+sanposhiho@users.noreply.github.com> Date: Fri, 4 Mar 2022 02:00:00 +0900 Subject: [PATCH] change framework.RunScorePlugins to return slice organized by node --- pkg/scheduler/framework/interface.go | 26 +++- pkg/scheduler/framework/runtime/framework.go | 45 ++++--- .../framework/runtime/framework_test.go | 125 ++++++++++++++++-- pkg/scheduler/schedule_one.go | 18 +-- 4 files changed, 167 insertions(+), 47 deletions(-) diff --git a/pkg/scheduler/framework/interface.go b/pkg/scheduler/framework/interface.go index 10c2776ddb6..7fa3a5f6d8e 100644 --- a/pkg/scheduler/framework/interface.go +++ b/pkg/scheduler/framework/interface.go @@ -48,12 +48,26 @@ type NodeScore struct { Score int64 } -// PluginToNodeScores declares a map from plugin name to its NodeScoreList. -type PluginToNodeScores map[string]NodeScoreList - // NodeToStatusMap declares map from node name to its status. type NodeToStatusMap map[string]*Status +// NodePluginScores is a struct with node name and scores for that node. +type NodePluginScores struct { + // Name is node name. + Name string + // Scores is scores from plugins and extenders. + Scores []PluginScore + // TotalScore is the total score in Scores. + TotalScore int64 +} + +// PluginScore is a struct with plugin/extender name and score. +type PluginScore struct { + // Name is the name of plugin or extender. + Name string + Score int64 +} + // Code is the Status code/type which is returned from plugins. type Code int @@ -709,11 +723,11 @@ type PluginsRunner interface { // RunPreScorePlugins runs the set of configured PreScore plugins. If any // of these plugins returns any status other than "Success", the given pod is rejected. RunPreScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) *Status - // RunScorePlugins runs the set of configured Score plugins. It returns a map that - // stores for each Score plugin name the corresponding NodeScoreList(s). + // RunScorePlugins runs the set of configured scoring plugins. + // It returns a list that stores scores from each plugin and total score for each Node. // It also returns *Status, which is set to non-success if any of the plugins returns // a non-success status. - RunScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) (PluginToNodeScores, *Status) + RunScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) ([]NodePluginScores, *Status) // RunFilterPlugins runs the set of configured Filter plugins for pod on // the given node. Note that for the node being evaluated, the passed nodeInfo // reference could be different from the one in NodeInfoSnapshot map (e.g., pods diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 6ff81c6f947..282f7c1a8d8 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -893,16 +893,17 @@ func (f *frameworkImpl) runPreScorePlugin(ctx context.Context, pl framework.PreS return status } -// RunScorePlugins runs the set of configured scoring plugins. It returns a list that -// stores for each scoring plugin name the corresponding NodeScoreList(s). +// RunScorePlugins runs the set of configured scoring plugins. +// It returns a list that stores scores from each plugin and total score for each Node. // It also returns *Status, which is set to non-success if any of the plugins returns // a non-success status. -func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) (ps framework.PluginToNodeScores, status *framework.Status) { +func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) (ns []framework.NodePluginScores, status *framework.Status) { startTime := time.Now() defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(score, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() - pluginToNodeScores := make(framework.PluginToNodeScores, len(f.scorePlugins)) + allNodePluginScores := make([]framework.NodePluginScores, len(nodes)) + pluginToNodeScores := make(map[string]framework.NodeScoreList, len(f.scorePlugins)) for _, pl := range f.scorePlugins { pluginToNodeScores[pl.Name()] = make(framework.NodeScoreList, len(nodes)) } @@ -933,10 +934,10 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy // Run NormalizeScore method for each ScorePlugin in parallel. f.Parallelizer().Until(ctx, len(f.scorePlugins), func(index int) { pl := f.scorePlugins[index] - nodeScoreList := pluginToNodeScores[pl.Name()] if pl.ScoreExtensions() == nil { return } + nodeScoreList := pluginToNodeScores[pl.Name()] status := f.runScoreExtension(ctx, pl, state, pod, nodeScoreList) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) @@ -948,28 +949,38 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy return nil, framework.AsStatus(fmt.Errorf("running Normalize on Score plugins: %w", err)) } - // Apply score defaultWeights for each ScorePlugin in parallel. - f.Parallelizer().Until(ctx, len(f.scorePlugins), func(index int) { - pl := f.scorePlugins[index] - // Score plugins' weight has been checked when they are initialized. - weight := f.scorePluginWeight[pl.Name()] - nodeScoreList := pluginToNodeScores[pl.Name()] + // Apply score weight for each ScorePlugin in parallel, + // and then, build allNodePluginScores. + f.Parallelizer().Until(ctx, len(nodes), func(index int) { + nodePluginScores := framework.NodePluginScores{ + Name: nodes[index].Name, + Scores: make([]framework.PluginScore, len(f.scorePlugins)), + } - for i, nodeScore := range nodeScoreList { - // return error if score plugin returns invalid score. - if nodeScore.Score > framework.MaxNodeScore || nodeScore.Score < framework.MinNodeScore { - err := fmt.Errorf("plugin %q returns an invalid score %v, it should in the range of [%v, %v] after normalizing", pl.Name(), nodeScore.Score, framework.MinNodeScore, framework.MaxNodeScore) + for i, pl := range f.scorePlugins { + weight := f.scorePluginWeight[pl.Name()] + nodeScoreList := pluginToNodeScores[pl.Name()] + score := nodeScoreList[index].Score + + if score > framework.MaxNodeScore || score < framework.MinNodeScore { + err := fmt.Errorf("plugin %q returns an invalid score %v, it should in the range of [%v, %v] after normalizing", pl.Name(), score, framework.MinNodeScore, framework.MaxNodeScore) errCh.SendErrorWithCancel(err, cancel) return } - nodeScoreList[i].Score = nodeScore.Score * int64(weight) + weightedScore := score * int64(weight) + nodePluginScores.Scores[i] = framework.PluginScore{ + Name: pl.Name(), + Score: weightedScore, + } + nodePluginScores.TotalScore += weightedScore } + allNodePluginScores[index] = nodePluginScores }, score) if err := errCh.ReceiveError(); err != nil { return nil, framework.AsStatus(fmt.Errorf("applying score defaultWeights on Score plugins: %w", err)) } - return pluginToNodeScores, nil + return allNodePluginScores, nil } func (f *frameworkImpl) runScorePlugin(ctx context.Context, pl framework.ScorePlugin, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) { diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go index 31cdcc1e941..7220c50a2db 100644 --- a/pkg/scheduler/framework/runtime/framework_test.go +++ b/pkg/scheduler/framework/runtime/framework_test.go @@ -988,14 +988,23 @@ func TestRunScorePlugins(t *testing.T) { registry Registry plugins *config.Plugins pluginConfigs []config.PluginConfig - want framework.PluginToNodeScores + want []framework.NodePluginScores // If err is true, we expect RunScorePlugin to fail. err bool }{ { name: "no Score plugins", plugins: buildScoreConfigDefaultWeights(), - want: framework.PluginToNodeScores{}, + want: []framework.NodePluginScores{ + { + Name: "node1", + Scores: []framework.PluginScore{}, + }, + { + Name: "node2", + Scores: []framework.PluginScore{}, + }, + }, }, { name: "single Score plugin", @@ -1009,8 +1018,27 @@ func TestRunScorePlugins(t *testing.T) { }, }, // scorePlugin1 Score returns 1, weight=1, so want=1. - want: framework.PluginToNodeScores{ - scorePlugin1: {{Name: "node1", Score: 1}, {Name: "node2", Score: 1}}, + want: []framework.NodePluginScores{ + { + Name: "node1", + Scores: []framework.PluginScore{ + { + Name: scorePlugin1, + Score: 1, + }, + }, + TotalScore: 1, + }, + { + Name: "node2", + Scores: []framework.PluginScore{ + { + Name: scorePlugin1, + Score: 1, + }, + }, + TotalScore: 1, + }, }, }, { @@ -1026,12 +1054,31 @@ func TestRunScorePlugins(t *testing.T) { }, }, // scoreWithNormalizePlugin1 Score returns 10, but NormalizeScore overrides to 5, weight=1, so want=5 - want: framework.PluginToNodeScores{ - scoreWithNormalizePlugin1: {{Name: "node1", Score: 5}, {Name: "node2", Score: 5}}, + want: []framework.NodePluginScores{ + { + Name: "node1", + Scores: []framework.PluginScore{ + { + Name: scoreWithNormalizePlugin1, + Score: 5, + }, + }, + TotalScore: 5, + }, + { + Name: "node2", + Scores: []framework.PluginScore{ + { + Name: scoreWithNormalizePlugin1, + Score: 5, + }, + }, + TotalScore: 5, + }, }, }, { - name: "2 Score plugins, 2 NormalizeScore plugins", + name: "3 Score plugins, 2 NormalizeScore plugins", plugins: buildScoreConfigDefaultWeights(scorePlugin1, scoreWithNormalizePlugin1, scoreWithNormalizePlugin2), pluginConfigs: []config.PluginConfig{ { @@ -1056,10 +1103,43 @@ func TestRunScorePlugins(t *testing.T) { // scorePlugin1 Score returns 1, weight =1, so want=1. // scoreWithNormalizePlugin1 Score returns 3, but NormalizeScore overrides to 4, weight=1, so want=4. // scoreWithNormalizePlugin2 Score returns 4, but NormalizeScore overrides to 5, weight=2, so want=10. - want: framework.PluginToNodeScores{ - scorePlugin1: {{Name: "node1", Score: 1}, {Name: "node2", Score: 1}}, - scoreWithNormalizePlugin1: {{Name: "node1", Score: 4}, {Name: "node2", Score: 4}}, - scoreWithNormalizePlugin2: {{Name: "node1", Score: 10}, {Name: "node2", Score: 10}}, + want: []framework.NodePluginScores{ + { + Name: "node1", + Scores: []framework.PluginScore{ + { + Name: scorePlugin1, + Score: 1, + }, + { + Name: scoreWithNormalizePlugin1, + Score: 4, + }, + { + Name: scoreWithNormalizePlugin2, + Score: 10, + }, + }, + TotalScore: 15, + }, + { + Name: "node2", + Scores: []framework.PluginScore{ + { + Name: scorePlugin1, + Score: 1, + }, + { + Name: scoreWithNormalizePlugin1, + Score: 4, + }, + { + Name: scoreWithNormalizePlugin2, + Score: 10, + }, + }, + TotalScore: 15, + }, }, }, { @@ -1163,8 +1243,27 @@ func TestRunScorePlugins(t *testing.T) { }, }, // scorePlugin1 Score returns 1, weight=3, so want=3. - want: framework.PluginToNodeScores{ - scorePlugin1: {{Name: "node1", Score: 3}, {Name: "node2", Score: 3}}, + want: []framework.NodePluginScores{ + { + Name: "node1", + Scores: []framework.PluginScore{ + { + Name: scorePlugin1, + Score: 3, + }, + }, + TotalScore: 3, + }, + { + Name: "node2", + Scores: []framework.PluginScore{ + { + Name: scorePlugin1, + Score: 3, + }, + }, + TotalScore: 3, + }, }, }, } diff --git a/pkg/scheduler/schedule_one.go b/pkg/scheduler/schedule_one.go index 133c81b38f1..1d449c31f89 100644 --- a/pkg/scheduler/schedule_one.go +++ b/pkg/scheduler/schedule_one.go @@ -684,7 +684,7 @@ func prioritizeNodes( } // Run the Score plugins. - scoresMap, scoreStatus := fwk.RunScorePlugins(ctx, state, pod, nodes) + nodesScores, scoreStatus := fwk.RunScorePlugins(ctx, state, pod, nodes) if !scoreStatus.IsSuccess() { return nil, scoreStatus.AsError() } @@ -692,21 +692,17 @@ func prioritizeNodes( // Additional details logged at level 10 if enabled. klogV := klog.V(10) if klogV.Enabled() { - for plugin, nodeScoreList := range scoresMap { - for _, nodeScore := range nodeScoreList { - klogV.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score) + for _, nodeScore := range nodesScores { + for _, pluginScore := range nodeScore.Scores { + klogV.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", pluginScore.Name, "node", nodeScore.Name, "score", pluginScore.Score) } } } // Summarize all scores. - result := make(framework.NodeScoreList, 0, len(nodes)) - - for i := range nodes { - result = append(result, framework.NodeScore{Name: nodes[i].Name, Score: 0}) - for j := range scoresMap { - result[i].Score += scoresMap[j][i].Score - } + result := make(framework.NodeScoreList, len(nodes)) + for i, pluginScores := range nodesScores { + result[i] = framework.NodeScore{Name: nodes[i].Name, Score: pluginScores.TotalScore} } if len(extenders) != 0 && nodes != nil {