change framework.RunScorePlugins to return slice organized by node

pkg/scheduler/framework/interface.go

@@ -48,12 +48,26 @@ type NodeScore struct {
 	Score int64
 }
 
-// PluginToNodeScores declares a map from plugin name to its NodeScoreList.
-type PluginToNodeScores map[string]NodeScoreList
-
 // NodeToStatusMap declares map from node name to its status.
 type NodeToStatusMap map[string]*Status
 
+// NodePluginScores is a struct with node name and scores for that node.
+type NodePluginScores struct {
+	// Name is node name.
+	Name string
+	// Scores is scores from plugins and extenders.
+	Scores []PluginScore
+	// TotalScore is the total score in Scores.
+	TotalScore int64
+}
+
+// PluginScore is a struct with plugin/extender name and score.
+type PluginScore struct {
+	// Name is the name of plugin or extender.
+	Name  string
+	Score int64
+}
+
 // Code is the Status code/type which is returned from plugins.
 type Code int
 
@@ -709,11 +723,11 @@ type PluginsRunner interface {
 	// RunPreScorePlugins runs the set of configured PreScore plugins. If any
 	// of these plugins returns any status other than "Success", the given pod is rejected.
 	RunPreScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) *Status
-	// RunScorePlugins runs the set of configured Score plugins. It returns a map that
+	// RunScorePlugins runs the set of configured scoring plugins.
-	// stores for each Score plugin name the corresponding NodeScoreList(s).
+	// It returns a list that stores scores from each plugin and total score for each Node.
 	// It also returns *Status, which is set to non-success if any of the plugins returns
 	// a non-success status.
-	RunScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) (PluginToNodeScores, *Status)
+	RunScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) ([]NodePluginScores, *Status)
 	// RunFilterPlugins runs the set of configured Filter plugins for pod on
 	// the given node. Note that for the node being evaluated, the passed nodeInfo
 	// reference could be different from the one in NodeInfoSnapshot map (e.g., pods

pkg/scheduler/framework/runtime/framework.go

@@ -893,16 +893,17 @@ func (f *frameworkImpl) runPreScorePlugin(ctx context.Context, pl framework.PreS
 	return status
 }
 
-// RunScorePlugins runs the set of configured scoring plugins. It returns a list that
+// RunScorePlugins runs the set of configured scoring plugins.
-// stores for each scoring plugin name the corresponding NodeScoreList(s).
+// It returns a list that stores scores from each plugin and total score for each Node.
 // It also returns *Status, which is set to non-success if any of the plugins returns
 // a non-success status.
-func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) (ps framework.PluginToNodeScores, status *framework.Status) {
+func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) (ns []framework.NodePluginScores, status *framework.Status) {
 	startTime := time.Now()
 	defer func() {
 		metrics.FrameworkExtensionPointDuration.WithLabelValues(score, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime))
 	}()
-	pluginToNodeScores := make(framework.PluginToNodeScores, len(f.scorePlugins))
+	allNodePluginScores := make([]framework.NodePluginScores, len(nodes))
+	pluginToNodeScores := make(map[string]framework.NodeScoreList, len(f.scorePlugins))
 	for _, pl := range f.scorePlugins {
 		pluginToNodeScores[pl.Name()] = make(framework.NodeScoreList, len(nodes))
 	}
@@ -933,10 +934,10 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy
 	// Run NormalizeScore method for each ScorePlugin in parallel.
 	f.Parallelizer().Until(ctx, len(f.scorePlugins), func(index int) {
 		pl := f.scorePlugins[index]
-		nodeScoreList := pluginToNodeScores[pl.Name()]
 		if pl.ScoreExtensions() == nil {
 			return
 		}
+		nodeScoreList := pluginToNodeScores[pl.Name()]
 		status := f.runScoreExtension(ctx, pl, state, pod, nodeScoreList)
 		if !status.IsSuccess() {
 			err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError())
@@ -948,28 +949,38 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy
 		return nil, framework.AsStatus(fmt.Errorf("running Normalize on Score plugins: %w", err))
 	}
 
-	// Apply score defaultWeights for each ScorePlugin in parallel.
+	// Apply score weight for each ScorePlugin in parallel,
-	f.Parallelizer().Until(ctx, len(f.scorePlugins), func(index int) {
+	// and then, build allNodePluginScores.
-		pl := f.scorePlugins[index]
+	f.Parallelizer().Until(ctx, len(nodes), func(index int) {
-		// Score plugins' weight has been checked when they are initialized.
+		nodePluginScores := framework.NodePluginScores{
-		weight := f.scorePluginWeight[pl.Name()]
+			Name:   nodes[index].Name,
-		nodeScoreList := pluginToNodeScores[pl.Name()]
+			Scores: make([]framework.PluginScore, len(f.scorePlugins)),
+		}
 
-		for i, nodeScore := range nodeScoreList {
+		for i, pl := range f.scorePlugins {
-			// return error if score plugin returns invalid score.
+			weight := f.scorePluginWeight[pl.Name()]
-			if nodeScore.Score > framework.MaxNodeScore || nodeScore.Score < framework.MinNodeScore {
+			nodeScoreList := pluginToNodeScores[pl.Name()]
-				err := fmt.Errorf("plugin %q returns an invalid score %v, it should in the range of [%v, %v] after normalizing", pl.Name(), nodeScore.Score, framework.MinNodeScore, framework.MaxNodeScore)
+			score := nodeScoreList[index].Score
+
+			if score > framework.MaxNodeScore || score < framework.MinNodeScore {
+				err := fmt.Errorf("plugin %q returns an invalid score %v, it should in the range of [%v, %v] after normalizing", pl.Name(), score, framework.MinNodeScore, framework.MaxNodeScore)
 				errCh.SendErrorWithCancel(err, cancel)
 				return
 			}
-			nodeScoreList[i].Score = nodeScore.Score * int64(weight)
+			weightedScore := score * int64(weight)
+			nodePluginScores.Scores[i] = framework.PluginScore{
+				Name:  pl.Name(),
+				Score: weightedScore,
+			}
+			nodePluginScores.TotalScore += weightedScore
 		}
+		allNodePluginScores[index] = nodePluginScores
 	}, score)
 	if err := errCh.ReceiveError(); err != nil {
 		return nil, framework.AsStatus(fmt.Errorf("applying score defaultWeights on Score plugins: %w", err))
 	}
 
-	return pluginToNodeScores, nil
+	return allNodePluginScores, nil
 }
 
 func (f *frameworkImpl) runScorePlugin(ctx context.Context, pl framework.ScorePlugin, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {

pkg/scheduler/framework/runtime/framework_test.go

@@ -988,14 +988,23 @@ func TestRunScorePlugins(t *testing.T) {
 		registry      Registry
 		plugins       *config.Plugins
 		pluginConfigs []config.PluginConfig
-		want          framework.PluginToNodeScores
+		want          []framework.NodePluginScores
 		// If err is true, we expect RunScorePlugin to fail.
 		err bool
 	}{
 		{
 			name:    "no Score plugins",
 			plugins: buildScoreConfigDefaultWeights(),
-			want:    framework.PluginToNodeScores{},
+			want: []framework.NodePluginScores{
+				{
+					Name:   "node1",
+					Scores: []framework.PluginScore{},
+				},
+				{
+					Name:   "node2",
+					Scores: []framework.PluginScore{},
+				},
+			},
 		},
 		{
 			name:    "single Score plugin",
@@ -1009,8 +1018,27 @@ func TestRunScorePlugins(t *testing.T) {
 				},
 			},
 			// scorePlugin1 Score returns 1, weight=1, so want=1.
-			want: framework.PluginToNodeScores{
+			want: []framework.NodePluginScores{
-				scorePlugin1: {{Name: "node1", Score: 1}, {Name: "node2", Score: 1}},
+				{
+					Name: "node1",
+					Scores: []framework.PluginScore{
+						{
+							Name:  scorePlugin1,
+							Score: 1,
+						},
+					},
+					TotalScore: 1,
+				},
+				{
+					Name: "node2",
+					Scores: []framework.PluginScore{
+						{
+							Name:  scorePlugin1,
+							Score: 1,
+						},
+					},
+					TotalScore: 1,
+				},
 			},
 		},
 		{
@@ -1026,12 +1054,31 @@ func TestRunScorePlugins(t *testing.T) {
 				},
 			},
 			// scoreWithNormalizePlugin1 Score returns 10, but NormalizeScore overrides to 5, weight=1, so want=5
-			want: framework.PluginToNodeScores{
+			want: []framework.NodePluginScores{
-				scoreWithNormalizePlugin1: {{Name: "node1", Score: 5}, {Name: "node2", Score: 5}},
+				{
+					Name: "node1",
+					Scores: []framework.PluginScore{
+						{
+							Name:  scoreWithNormalizePlugin1,
+							Score: 5,
+						},
+					},
+					TotalScore: 5,
+				},
+				{
+					Name: "node2",
+					Scores: []framework.PluginScore{
+						{
+							Name:  scoreWithNormalizePlugin1,
+							Score: 5,
+						},
+					},
+					TotalScore: 5,
+				},
 			},
 		},
 		{
-			name:    "2 Score plugins, 2 NormalizeScore plugins",
+			name:    "3 Score plugins, 2 NormalizeScore plugins",
 			plugins: buildScoreConfigDefaultWeights(scorePlugin1, scoreWithNormalizePlugin1, scoreWithNormalizePlugin2),
 			pluginConfigs: []config.PluginConfig{
 				{
@@ -1056,10 +1103,43 @@ func TestRunScorePlugins(t *testing.T) {
 			// scorePlugin1 Score returns 1, weight =1, so want=1.
 			// scoreWithNormalizePlugin1 Score returns 3, but NormalizeScore overrides to 4, weight=1, so want=4.
 			// scoreWithNormalizePlugin2 Score returns 4, but NormalizeScore overrides to 5, weight=2, so want=10.
-			want: framework.PluginToNodeScores{
+			want: []framework.NodePluginScores{
-				scorePlugin1:              {{Name: "node1", Score: 1}, {Name: "node2", Score: 1}},
+				{
-				scoreWithNormalizePlugin1: {{Name: "node1", Score: 4}, {Name: "node2", Score: 4}},
+					Name: "node1",
-				scoreWithNormalizePlugin2: {{Name: "node1", Score: 10}, {Name: "node2", Score: 10}},
+					Scores: []framework.PluginScore{
+						{
+							Name:  scorePlugin1,
+							Score: 1,
+						},
+						{
+							Name:  scoreWithNormalizePlugin1,
+							Score: 4,
+						},
+						{
+							Name:  scoreWithNormalizePlugin2,
+							Score: 10,
+						},
+					},
+					TotalScore: 15,
+				},
+				{
+					Name: "node2",
+					Scores: []framework.PluginScore{
+						{
+							Name:  scorePlugin1,
+							Score: 1,
+						},
+						{
+							Name:  scoreWithNormalizePlugin1,
+							Score: 4,
+						},
+						{
+							Name:  scoreWithNormalizePlugin2,
+							Score: 10,
+						},
+					},
+					TotalScore: 15,
+				},
 			},
 		},
 		{
@@ -1163,8 +1243,27 @@ func TestRunScorePlugins(t *testing.T) {
 				},
 			},
 			// scorePlugin1 Score returns 1, weight=3, so want=3.
-			want: framework.PluginToNodeScores{
+			want: []framework.NodePluginScores{
-				scorePlugin1: {{Name: "node1", Score: 3}, {Name: "node2", Score: 3}},
+				{
+					Name: "node1",
+					Scores: []framework.PluginScore{
+						{
+							Name:  scorePlugin1,
+							Score: 3,
+						},
+					},
+					TotalScore: 3,
+				},
+				{
+					Name: "node2",
+					Scores: []framework.PluginScore{
+						{
+							Name:  scorePlugin1,
+							Score: 3,
+						},
+					},
+					TotalScore: 3,
+				},
 			},
 		},
 	}

pkg/scheduler/schedule_one.go

@@ -684,7 +684,7 @@ func prioritizeNodes(
 	}
 
 	// Run the Score plugins.
-	scoresMap, scoreStatus := fwk.RunScorePlugins(ctx, state, pod, nodes)
+	nodesScores, scoreStatus := fwk.RunScorePlugins(ctx, state, pod, nodes)
 	if !scoreStatus.IsSuccess() {
 		return nil, scoreStatus.AsError()
 	}
@@ -692,21 +692,17 @@ func prioritizeNodes(
 	// Additional details logged at level 10 if enabled.
 	klogV := klog.V(10)
 	if klogV.Enabled() {
-		for plugin, nodeScoreList := range scoresMap {
+		for _, nodeScore := range nodesScores {
-			for _, nodeScore := range nodeScoreList {
+			for _, pluginScore := range nodeScore.Scores {
-				klogV.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score)
+				klogV.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", pluginScore.Name, "node", nodeScore.Name, "score", pluginScore.Score)
 			}
 		}
 	}
 
 	// Summarize all scores.
-	result := make(framework.NodeScoreList, 0, len(nodes))
+	result := make(framework.NodeScoreList, len(nodes))
-
+	for i, pluginScores := range nodesScores {
-	for i := range nodes {
+		result[i] = framework.NodeScore{Name: nodes[i].Name, Score: pluginScores.TotalScore}
-		result = append(result, framework.NodeScore{Name: nodes[i].Name, Score: 0})
-		for j := range scoresMap {
-			result[i].Score += scoresMap[j][i].Score
-		}
 	}
 
 	if len(extenders) != 0 && nodes != nil {