github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 10:51:29 +00:00
Code Issues Projects Releases Wiki Activity

change framework.RunScorePlugins to return slice organized by node

Browse Source
This commit is contained in:
sanposhiho 2022-03-04 02:00:00 +09:00 committed by Kensei Nakada
parent 6820a383be
commit cbf1ea5e68
4 changed files with 167 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
pkg/scheduler/framework/interface.go
View File

@ -48,12 +48,26 @@ type NodeScore struct {
Score int64
}
// PluginToNodeScores declares a map from plugin name to its NodeScoreList.
type PluginToNodeScores map[string]NodeScoreList
// NodeToStatusMap declares map from node name to its status.
type NodeToStatusMap map[string]*Status
// NodePluginScores is a struct with node name and scores for that node.
type NodePluginScores struct {
// Name is node name.
Name string
// Scores is scores from plugins and extenders.
Scores []PluginScore
// TotalScore is the total score in Scores.
TotalScore int64
}
// PluginScore is a struct with plugin/extender name and score.
type PluginScore struct {
// Name is the name of plugin or extender.
Name string
Score int64
}
// Code is the Status code/type which is returned from plugins.
type Code int
@ -709,11 +723,11 @@ type PluginsRunner interface {
// RunPreScorePlugins runs the set of configured PreScore plugins. If any
// of these plugins returns any status other than "Success", the given pod is rejected.
RunPreScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) *Status
// RunScorePlugins runs the set of configured Score plugins. It returns a map that
// stores for each Score plugin name the corresponding NodeScoreList(s).
// RunScorePlugins runs the set of configured scoring plugins.
// It returns a list that stores scores from each plugin and total score for each Node.
// It also returns *Status, which is set to non-success if any of the plugins returns
// a non-success status.
RunScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) (PluginToNodeScores, *Status)
RunScorePlugins(context.Context, *CycleState, *v1.Pod, []*v1.Node) ([]NodePluginScores, *Status)
// RunFilterPlugins runs the set of configured Filter plugins for pod on
// the given node. Note that for the node being evaluated, the passed nodeInfo
// reference could be different from the one in NodeInfoSnapshot map (e.g., pods

45
pkg/scheduler/framework/runtime/framework.go
View File

@ -893,16 +893,17 @@ func (f *frameworkImpl) runPreScorePlugin(ctx context.Context, pl framework.PreS
return status
}
// RunScorePlugins runs the set of configured scoring plugins. It returns a list that
// stores for each scoring plugin name the corresponding NodeScoreList(s).
// RunScorePlugins runs the set of configured scoring plugins.
// It returns a list that stores scores from each plugin and total score for each Node.
// It also returns *Status, which is set to non-success if any of the plugins returns
// a non-success status.
func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) (ps framework.PluginToNodeScores, status *framework.Status) {
func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) (ns []framework.NodePluginScores, status *framework.Status) {
startTime := time.Now()
defer func() {
metrics.FrameworkExtensionPointDuration.WithLabelValues(score, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime))
}()
pluginToNodeScores := make(framework.PluginToNodeScores, len(f.scorePlugins))
allNodePluginScores := make([]framework.NodePluginScores, len(nodes))
pluginToNodeScores := make(map[string]framework.NodeScoreList, len(f.scorePlugins))
for _, pl := range f.scorePlugins {
pluginToNodeScores[pl.Name()] = make(framework.NodeScoreList, len(nodes))
}
@ -933,10 +934,10 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy
// Run NormalizeScore method for each ScorePlugin in parallel.
f.Parallelizer().Until(ctx, len(f.scorePlugins), func(index int) {
pl := f.scorePlugins[index]
nodeScoreList := pluginToNodeScores[pl.Name()]
if pl.ScoreExtensions() == nil {
return
}
nodeScoreList := pluginToNodeScores[pl.Name()]
status := f.runScoreExtension(ctx, pl, state, pod, nodeScoreList)
if !status.IsSuccess() {
err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError())
@ -948,28 +949,38 @@ func (f *frameworkImpl) RunScorePlugins(ctx context.Context, state *framework.Cy
return nil, framework.AsStatus(fmt.Errorf("running Normalize on Score plugins: %w", err))
}
// Apply score defaultWeights for each ScorePlugin in parallel.
f.Parallelizer().Until(ctx, len(f.scorePlugins), func(index int) {
pl := f.scorePlugins[index]
// Score plugins' weight has been checked when they are initialized.
weight := f.scorePluginWeight[pl.Name()]
nodeScoreList := pluginToNodeScores[pl.Name()]
// Apply score weight for each ScorePlugin in parallel,
// and then, build allNodePluginScores.
f.Parallelizer().Until(ctx, len(nodes), func(index int) {
nodePluginScores := framework.NodePluginScores{
Name: nodes[index].Name,
Scores: make([]framework.PluginScore, len(f.scorePlugins)),
}
for i, nodeScore := range nodeScoreList {
// return error if score plugin returns invalid score.
if nodeScore.Score > framework.MaxNodeScore || nodeScore.Score < framework.MinNodeScore {
err := fmt.Errorf("plugin %q returns an invalid score %v, it should in the range of [%v, %v] after normalizing", pl.Name(), nodeScore.Score, framework.MinNodeScore, framework.MaxNodeScore)
for i, pl := range f.scorePlugins {
weight := f.scorePluginWeight[pl.Name()]
nodeScoreList := pluginToNodeScores[pl.Name()]
score := nodeScoreList[index].Score
if score > framework.MaxNodeScore || score < framework.MinNodeScore {
err := fmt.Errorf("plugin %q returns an invalid score %v, it should in the range of [%v, %v] after normalizing", pl.Name(), score, framework.MinNodeScore, framework.MaxNodeScore)
errCh.SendErrorWithCancel(err, cancel)
return
}
nodeScoreList[i].Score = nodeScore.Score * int64(weight)
weightedScore := score * int64(weight)
nodePluginScores.Scores[i] = framework.PluginScore{
Name: pl.Name(),
Score: weightedScore,
}
nodePluginScores.TotalScore += weightedScore
}
allNodePluginScores[index] = nodePluginScores
}, score)
if err := errCh.ReceiveError(); err != nil {
return nil, framework.AsStatus(fmt.Errorf("applying score defaultWeights on Score plugins: %w", err))
}
return pluginToNodeScores, nil
return allNodePluginScores, nil
}
func (f *frameworkImpl) runScorePlugin(ctx context.Context, pl framework.ScorePlugin, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {

125
pkg/scheduler/framework/runtime/framework_test.go
View File

@ -988,14 +988,23 @@ func TestRunScorePlugins(t *testing.T) {
registry Registry
plugins *config.Plugins
pluginConfigs []config.PluginConfig
want framework.PluginToNodeScores
want []framework.NodePluginScores
// If err is true, we expect RunScorePlugin to fail.
err bool
}{
{
name: "no Score plugins",
plugins: buildScoreConfigDefaultWeights(),
want: framework.PluginToNodeScores{},
want: []framework.NodePluginScores{
{
Name: "node1",
Scores: []framework.PluginScore{},
},
{
Name: "node2",
Scores: []framework.PluginScore{},
},
},
},
{
name: "single Score plugin",
@ -1009,8 +1018,27 @@ func TestRunScorePlugins(t *testing.T) {
},
},
// scorePlugin1 Score returns 1, weight=1, so want=1.
want: framework.PluginToNodeScores{
scorePlugin1: {{Name: "node1", Score: 1}, {Name: "node2", Score: 1}},
want: []framework.NodePluginScores{
{
Name: "node1",
Scores: []framework.PluginScore{
{
Name: scorePlugin1,
Score: 1,
},
},
TotalScore: 1,
},
{
Name: "node2",
Scores: []framework.PluginScore{
{
Name: scorePlugin1,
Score: 1,
},
},
TotalScore: 1,
},
},
},
{
@ -1026,12 +1054,31 @@ func TestRunScorePlugins(t *testing.T) {
},
},
// scoreWithNormalizePlugin1 Score returns 10, but NormalizeScore overrides to 5, weight=1, so want=5
want: framework.PluginToNodeScores{
scoreWithNormalizePlugin1: {{Name: "node1", Score: 5}, {Name: "node2", Score: 5}},
want: []framework.NodePluginScores{
{
Name: "node1",
Scores: []framework.PluginScore{
{
Name: scoreWithNormalizePlugin1,
Score: 5,
},
},
TotalScore: 5,
},
{
Name: "node2",
Scores: []framework.PluginScore{
{
Name: scoreWithNormalizePlugin1,
Score: 5,
},
},
TotalScore: 5,
},
},
},
{
name: "2 Score plugins, 2 NormalizeScore plugins",
name: "3 Score plugins, 2 NormalizeScore plugins",
plugins: buildScoreConfigDefaultWeights(scorePlugin1, scoreWithNormalizePlugin1, scoreWithNormalizePlugin2),
pluginConfigs: []config.PluginConfig{
{
@ -1056,10 +1103,43 @@ func TestRunScorePlugins(t *testing.T) {
// scorePlugin1 Score returns 1, weight =1, so want=1.
// scoreWithNormalizePlugin1 Score returns 3, but NormalizeScore overrides to 4, weight=1, so want=4.
// scoreWithNormalizePlugin2 Score returns 4, but NormalizeScore overrides to 5, weight=2, so want=10.
want: framework.PluginToNodeScores{
scorePlugin1: {{Name: "node1", Score: 1}, {Name: "node2", Score: 1}},
scoreWithNormalizePlugin1: {{Name: "node1", Score: 4}, {Name: "node2", Score: 4}},
scoreWithNormalizePlugin2: {{Name: "node1", Score: 10}, {Name: "node2", Score: 10}},
want: []framework.NodePluginScores{
{
Name: "node1",
Scores: []framework.PluginScore{
{
Name: scorePlugin1,
Score: 1,
},
{
Name: scoreWithNormalizePlugin1,
Score: 4,
},
{
Name: scoreWithNormalizePlugin2,
Score: 10,
},
},
TotalScore: 15,
},
{
Name: "node2",
Scores: []framework.PluginScore{
{
Name: scorePlugin1,
Score: 1,
},
{
Name: scoreWithNormalizePlugin1,
Score: 4,
},
{
Name: scoreWithNormalizePlugin2,
Score: 10,
},
},
TotalScore: 15,
},
},
},
{
@ -1163,8 +1243,27 @@ func TestRunScorePlugins(t *testing.T) {
},
},
// scorePlugin1 Score returns 1, weight=3, so want=3.
want: framework.PluginToNodeScores{
scorePlugin1: {{Name: "node1", Score: 3}, {Name: "node2", Score: 3}},
want: []framework.NodePluginScores{
{
Name: "node1",
Scores: []framework.PluginScore{
{
Name: scorePlugin1,
Score: 3,
},
},
TotalScore: 3,
},
{
Name: "node2",
Scores: []framework.PluginScore{
{
Name: scorePlugin1,
Score: 3,
},
},
TotalScore: 3,
},
},
},
}

18
pkg/scheduler/schedule_one.go
View File

@ -684,7 +684,7 @@ func prioritizeNodes(
}
// Run the Score plugins.
scoresMap, scoreStatus := fwk.RunScorePlugins(ctx, state, pod, nodes)
nodesScores, scoreStatus := fwk.RunScorePlugins(ctx, state, pod, nodes)
if !scoreStatus.IsSuccess() {
return nil, scoreStatus.AsError()
}
@ -692,21 +692,17 @@ func prioritizeNodes(
// Additional details logged at level 10 if enabled.
klogV := klog.V(10)
if klogV.Enabled() {
for plugin, nodeScoreList := range scoresMap {
for _, nodeScore := range nodeScoreList {
klogV.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score)
for _, nodeScore := range nodesScores {
for _, pluginScore := range nodeScore.Scores {
klogV.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", pluginScore.Name, "node", nodeScore.Name, "score", pluginScore.Score)
}
}
}
// Summarize all scores.
result := make(framework.NodeScoreList, 0, len(nodes))
for i := range nodes {
result = append(result, framework.NodeScore{Name: nodes[i].Name, Score: 0})
for j := range scoresMap {
result[i].Score += scoresMap[j][i].Score
}
result := make(framework.NodeScoreList, len(nodes))
for i, pluginScores := range nodesScores {
result[i] = framework.NodeScore{Name: nodes[i].Name, Score: pluginScores.TotalScore}
}
if len(extenders) != 0 && nodes != nil {