From a4ab559ba4f20a6aa55825ca26af31818d6e57ef Mon Sep 17 00:00:00 2001 From: tangwz Date: Sat, 10 Dec 2022 02:11:27 +0800 Subject: [PATCH] Improve performance of NodeResourcesFit scoring Signed-off-by: tangwz --- .../noderesources/balanced_allocation.go | 23 ++-- .../framework/plugins/noderesources/fit.go | 24 ++-- .../plugins/noderesources/fit_test.go | 113 ++++++++++++++++++ .../plugins/noderesources/least_allocated.go | 14 ++- .../noderesources/least_allocated_test.go | 21 ++++ .../plugins/noderesources/most_allocated.go | 14 ++- .../noderesources/most_allocated_test.go | 21 ++++ .../requested_to_capacity_ratio.go | 17 +-- .../noderesources/resource_allocation.go | 40 +++---- 9 files changed, 219 insertions(+), 68 deletions(-) diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go index baad87699ac..4c0438bbc3d 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go @@ -78,28 +78,25 @@ func NewBalancedAllocation(baArgs runtime.Object, h framework.Handle, fts featur return nil, err } - resToWeightMap := make(resourceToWeightMap) - - for _, resource := range args.Resources { - resToWeightMap[v1.ResourceName(resource.Name)] = resource.Weight - } - return &BalancedAllocation{ handle: h, resourceAllocationScorer: resourceAllocationScorer{ - Name: BalancedAllocationName, - scorer: balancedResourceScorer, - useRequested: true, - resourceToWeightMap: resToWeightMap, + Name: BalancedAllocationName, + scorer: balancedResourceScorer, + useRequested: true, + resources: args.Resources, }, }, nil } -func balancedResourceScorer(requested, allocable resourceToValueMap) int64 { +func balancedResourceScorer(requested, allocable []int64) int64 { var resourceToFractions []float64 var totalFraction float64 - for name, value := range requested { - fraction := float64(value) / float64(allocable[name]) + for i := range requested { + if allocable[i] == 0 { + continue + } + fraction := float64(requested[i]) / float64(allocable[i]) if fraction > 1 { fraction = 1 } diff --git a/pkg/scheduler/framework/plugins/noderesources/fit.go b/pkg/scheduler/framework/plugins/noderesources/fit.go index ff62eff0041..27cb3233eef 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit.go @@ -49,27 +49,27 @@ const ( // nodeResourceStrategyTypeMap maps strategy to scorer implementation var nodeResourceStrategyTypeMap = map[config.ScoringStrategyType]scorer{ config.LeastAllocated: func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer { - resToWeightMap := resourcesToWeightMap(args.ScoringStrategy.Resources) + resources := args.ScoringStrategy.Resources return &resourceAllocationScorer{ - Name: string(config.LeastAllocated), - scorer: leastResourceScorer(resToWeightMap), - resourceToWeightMap: resToWeightMap, + Name: string(config.LeastAllocated), + scorer: leastResourceScorer(resources), + resources: resources, } }, config.MostAllocated: func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer { - resToWeightMap := resourcesToWeightMap(args.ScoringStrategy.Resources) + resources := args.ScoringStrategy.Resources return &resourceAllocationScorer{ - Name: string(config.MostAllocated), - scorer: mostResourceScorer(resToWeightMap), - resourceToWeightMap: resToWeightMap, + Name: string(config.MostAllocated), + scorer: mostResourceScorer(resources), + resources: resources, } }, config.RequestedToCapacityRatio: func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer { - resToWeightMap := resourcesToWeightMap(args.ScoringStrategy.Resources) + resources := args.ScoringStrategy.Resources return &resourceAllocationScorer{ - Name: string(config.RequestedToCapacityRatio), - scorer: requestedToCapacityRatioScorer(resToWeightMap, args.ScoringStrategy.RequestedToCapacityRatio.Shape), - resourceToWeightMap: resToWeightMap, + Name: string(config.RequestedToCapacityRatio), + scorer: requestedToCapacityRatioScorer(resources, args.ScoringStrategy.RequestedToCapacityRatio.Shape), + resources: resources, } }, } diff --git a/pkg/scheduler/framework/plugins/noderesources/fit_test.go b/pkg/scheduler/framework/plugins/noderesources/fit_test.go index 6477dbfce44..ffe5a26969b 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" plfeature "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" + plugintesting "k8s.io/kubernetes/pkg/scheduler/framework/plugins/testing" "k8s.io/kubernetes/pkg/scheduler/framework/runtime" "k8s.io/kubernetes/pkg/scheduler/internal/cache" st "k8s.io/kubernetes/pkg/scheduler/testing" @@ -780,3 +781,115 @@ func TestFitScore(t *testing.T) { }) } } + +var benchmarkResourceSet = []config.ResourceSpec{ + {Name: string(v1.ResourceCPU), Weight: 1}, + {Name: string(v1.ResourceMemory), Weight: 1}, + {Name: string(v1.ResourcePods), Weight: 1}, + {Name: string(v1.ResourceStorage), Weight: 1}, + {Name: string(v1.ResourceEphemeralStorage), Weight: 1}, + {Name: string(extendedResourceA), Weight: 1}, + {Name: string(extendedResourceB), Weight: 1}, + {Name: string(kubernetesIOResourceA), Weight: 1}, + {Name: string(kubernetesIOResourceB), Weight: 1}, + {Name: string(hugePageResourceA), Weight: 1}, +} + +func BenchmarkTestFitScore(b *testing.B) { + tests := []struct { + name string + nodeResourcesFitArgs config.NodeResourcesFitArgs + }{ + { + name: "RequestedToCapacityRatio with defaultResources", + nodeResourcesFitArgs: config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.RequestedToCapacityRatio, + Resources: defaultResources, + RequestedToCapacityRatio: &config.RequestedToCapacityRatioParam{ + Shape: []config.UtilizationShapePoint{ + {Utilization: 0, Score: 10}, + {Utilization: 100, Score: 0}, + }, + }, + }, + }, + }, + { + name: "RequestedToCapacityRatio with 10 resources", + nodeResourcesFitArgs: config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.RequestedToCapacityRatio, + Resources: benchmarkResourceSet, + RequestedToCapacityRatio: &config.RequestedToCapacityRatioParam{ + Shape: []config.UtilizationShapePoint{ + {Utilization: 0, Score: 10}, + {Utilization: 100, Score: 0}, + }, + }, + }, + }, + }, + { + name: "MostAllocated with defaultResources", + nodeResourcesFitArgs: config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.MostAllocated, + Resources: defaultResources, + }, + }, + }, + { + name: "MostAllocated with 10 resources", + nodeResourcesFitArgs: config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.MostAllocated, + Resources: benchmarkResourceSet, + }, + }, + }, + { + name: "LeastAllocated with defaultResources", + nodeResourcesFitArgs: config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.LeastAllocated, + Resources: defaultResources, + }, + }, + }, + { + name: "LeastAllocated with 10 resources", + nodeResourcesFitArgs: config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.LeastAllocated, + Resources: benchmarkResourceSet, + }, + }, + }, + } + + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + existingPods := []*v1.Pod{ + st.MakePod().Node("node1").Req(map[v1.ResourceName]string{"cpu": "2000", "memory": "4000"}).Obj(), + } + nodes := []*v1.Node{ + st.MakeNode().Name("node1").Capacity(map[v1.ResourceName]string{"cpu": "4000", "memory": "10000"}).Obj(), + } + state := framework.NewCycleState() + var nodeResourcesFunc = runtime.FactoryAdapter(plfeature.Features{}, NewFit) + pl := plugintesting.SetupPlugin(b, nodeResourcesFunc, &test.nodeResourcesFitArgs, cache.NewSnapshot(existingPods, nodes)) + p := pl.(*Fit) + + b.ResetTimer() + + requestedPod := st.MakePod().Req(map[v1.ResourceName]string{"cpu": "1000", "memory": "2000"}).Obj() + for i := 0; i < b.N; i++ { + _, status := p.Score(context.Background(), state, requestedPod, nodes[0].Name) + if !status.IsSuccess() { + b.Errorf("unexpected status: %v", status) + } + } + }) + } +} diff --git a/pkg/scheduler/framework/plugins/noderesources/least_allocated.go b/pkg/scheduler/framework/plugins/noderesources/least_allocated.go index 1abf97bc678..f571bcbc2a8 100644 --- a/pkg/scheduler/framework/plugins/noderesources/least_allocated.go +++ b/pkg/scheduler/framework/plugins/noderesources/least_allocated.go @@ -17,6 +17,7 @@ limitations under the License. package noderesources import ( + "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -26,12 +27,15 @@ import ( // // Details: // (cpu((capacity-requested)*MaxNodeScore*cpuWeight/capacity) + memory((capacity-requested)*MaxNodeScore*memoryWeight/capacity) + ...)/weightSum -func leastResourceScorer(resToWeightMap resourceToWeightMap) func(resourceToValueMap, resourceToValueMap) int64 { - return func(requested, allocable resourceToValueMap) int64 { +func leastResourceScorer(resources []config.ResourceSpec) func([]int64, []int64) int64 { + return func(requested, allocable []int64) int64 { var nodeScore, weightSum int64 - for resource := range requested { - weight := resToWeightMap[resource] - resourceScore := leastRequestedScore(requested[resource], allocable[resource]) + for i := range requested { + if allocable[i] == 0 { + continue + } + weight := resources[i].Weight + resourceScore := leastRequestedScore(requested[i], allocable[i]) nodeScore += resourceScore * weight weightSum += weight } diff --git a/pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go b/pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go index 30758e81534..43f52bf6004 100644 --- a/pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go @@ -360,6 +360,27 @@ func TestLeastAllocatedScoringStrategy(t *testing.T) { expectedScores: []framework.NodeScore{{Name: "node1", Score: 50}, {Name: "node2", Score: 60}}, resources: extendedResourceSet, }, + { + // If the node doesn't have a resource + // CPU Score: ((6000 - 3000) * 100) / 6000 = 50 + // Memory Score: ((10000 - 4000) * 100) / 10000 = 60 + // Node1 Score: (50 * 1 + 60 * 1) / (1 + 1) = 55 + // Node2 Score: (50 * 1 + 60 * 1) / (1 + 1) = 55 + name: "if the node doesn't have a resource", + requestedPod: st.MakePod().Node("node1"). + Req(map[v1.ResourceName]string{"cpu": "3000", "memory": "4000"}). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node1").Capacity(map[v1.ResourceName]string{"cpu": "6000", "memory": "10000"}).Obj(), + st.MakeNode().Name("node2").Capacity(map[v1.ResourceName]string{"cpu": "6000", "memory": "10000", v1.ResourceName(extendedRes): "4"}).Obj(), + }, + expectedScores: []framework.NodeScore{{Name: "node1", Score: 55}, {Name: "node2", Score: 55}}, + resources: []config.ResourceSpec{ + {Name: extendedRes, Weight: 2}, + {Name: string(v1.ResourceCPU), Weight: 1}, + {Name: string(v1.ResourceMemory), Weight: 1}, + }, + }, } for _, test := range tests { diff --git a/pkg/scheduler/framework/plugins/noderesources/most_allocated.go b/pkg/scheduler/framework/plugins/noderesources/most_allocated.go index 94c55cbe325..41425698835 100644 --- a/pkg/scheduler/framework/plugins/noderesources/most_allocated.go +++ b/pkg/scheduler/framework/plugins/noderesources/most_allocated.go @@ -17,6 +17,7 @@ limitations under the License. package noderesources import ( + "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -26,12 +27,15 @@ import ( // // Details: // (cpu(MaxNodeScore * requested * cpuWeight / capacity) + memory(MaxNodeScore * requested * memoryWeight / capacity) + ...) / weightSum -func mostResourceScorer(resToWeightMap resourceToWeightMap) func(requested, allocable resourceToValueMap) int64 { - return func(requested, allocable resourceToValueMap) int64 { +func mostResourceScorer(resources []config.ResourceSpec) func(requested, allocable []int64) int64 { + return func(requested, allocable []int64) int64 { var nodeScore, weightSum int64 - for resource := range requested { - weight := resToWeightMap[resource] - resourceScore := mostRequestedScore(requested[resource], allocable[resource]) + for i := range requested { + if allocable[i] == 0 { + continue + } + weight := resources[i].Weight + resourceScore := mostRequestedScore(requested[i], allocable[i]) nodeScore += resourceScore * weight weightSum += weight } diff --git a/pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go b/pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go index b3e289077e4..d7f52a6fe7b 100644 --- a/pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go @@ -317,6 +317,27 @@ func TestMostAllocatedScoringStrategy(t *testing.T) { existingPods: nil, expectedScores: []framework.NodeScore{{Name: "node1", Score: 50}, {Name: "node2", Score: 40}}, }, + { + // If the node doesn't have a resource + // CPU Score: (3000 * 100) / 6000 = 50 + // Memory Score: (4000 * 100) / 10000 = 40 + // Node1 Score: (50 * 1 + 40 * 1) / (1 + 1) = 45 + // Node2 Score: (50 * 1 + 40 * 1) / (1 + 1) = 45 + name: "if the node doesn't have a resource", + requestedPod: st.MakePod().Node("node1"). + Req(map[v1.ResourceName]string{"cpu": "3000", "memory": "4000"}). + Obj(), + nodes: []*v1.Node{ + st.MakeNode().Name("node1").Capacity(map[v1.ResourceName]string{"cpu": "6000", "memory": "10000"}).Obj(), + st.MakeNode().Name("node2").Capacity(map[v1.ResourceName]string{"cpu": "6000", "memory": "10000", v1.ResourceName(extendedRes): "4"}).Obj(), + }, + expectedScores: []framework.NodeScore{{Name: "node1", Score: 45}, {Name: "node2", Score: 45}}, + resources: []config.ResourceSpec{ + {Name: extendedRes, Weight: 2}, + {Name: string(v1.ResourceCPU), Weight: 1}, + {Name: string(v1.ResourceMemory), Weight: 1}, + }, + }, } for _, test := range tests { diff --git a/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go b/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go index 36cd9f1e623..df5e2ff4733 100644 --- a/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go +++ b/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go @@ -28,7 +28,7 @@ const maxUtilization = 100 // buildRequestedToCapacityRatioScorerFunction allows users to apply bin packing // on core resources like CPU, Memory as well as extended resources like accelerators. -func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resourceToWeightMap resourceToWeightMap) func(resourceToValueMap, resourceToValueMap) int64 { +func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resources []config.ResourceSpec) func([]int64, []int64) int64 { rawScoringFunction := helper.BuildBrokenLinearFunction(scoringFunctionShape) resourceScoringFunction := func(requested, capacity int64) int64 { if capacity == 0 || requested > capacity { @@ -37,11 +37,14 @@ func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.Fun return rawScoringFunction(requested * maxUtilization / capacity) } - return func(requested, allocable resourceToValueMap) int64 { + return func(requested, allocable []int64) int64 { var nodeScore, weightSum int64 - for resource := range requested { - weight := resourceToWeightMap[resource] - resourceScore := resourceScoringFunction(requested[resource], allocable[resource]) + for i := range requested { + if allocable[i] == 0 { + continue + } + weight := resources[i].Weight + resourceScore := resourceScoringFunction(requested[i], allocable[i]) if resourceScore > 0 { nodeScore += resourceScore * weight weightSum += weight @@ -54,7 +57,7 @@ func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.Fun } } -func requestedToCapacityRatioScorer(weightMap resourceToWeightMap, shape []config.UtilizationShapePoint) func(resourceToValueMap, resourceToValueMap) int64 { +func requestedToCapacityRatioScorer(resources []config.ResourceSpec, shape []config.UtilizationShapePoint) func([]int64, []int64) int64 { shapes := make([]helper.FunctionShapePoint, 0, len(shape)) for _, point := range shape { shapes = append(shapes, helper.FunctionShapePoint{ @@ -66,5 +69,5 @@ func requestedToCapacityRatioScorer(weightMap resourceToWeightMap, shape []confi }) } - return buildRequestedToCapacityRatioScorerFunction(shapes, weightMap) + return buildRequestedToCapacityRatioScorerFunction(shapes, resources) } diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go index 799de45ad73..af3e8b15ca9 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go @@ -24,9 +24,6 @@ import ( schedutil "k8s.io/kubernetes/pkg/scheduler/util" ) -// resourceToWeightMap contains resource name and weight. -type resourceToWeightMap map[v1.ResourceName]int64 - // scorer is decorator for resourceAllocationScorer type scorer func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer @@ -35,14 +32,11 @@ type resourceAllocationScorer struct { Name string // used to decide whether to use Requested or NonZeroRequested for // cpu and memory. - useRequested bool - scorer func(requested, allocable resourceToValueMap) int64 - resourceToWeightMap resourceToWeightMap + useRequested bool + scorer func(requested, allocable []int64) int64 + resources []config.ResourceSpec } -// resourceToValueMap is keyed with resource name and valued with quantity. -type resourceToValueMap map[v1.ResourceName]int64 - // score will use `scorer` function to calculate the score. func (r *resourceAllocationScorer) score( pod *v1.Pod, @@ -51,18 +45,21 @@ func (r *resourceAllocationScorer) score( if node == nil { return 0, framework.NewStatus(framework.Error, "node not found") } - if len(r.resourceToWeightMap) == 0 { + // resources not set, nothing scheduled, + if len(r.resources) == 0 { return 0, framework.NewStatus(framework.Error, "resources not found") } - requested := make(resourceToValueMap) - allocatable := make(resourceToValueMap) - for resource := range r.resourceToWeightMap { - alloc, req := r.calculateResourceAllocatableRequest(nodeInfo, pod, resource) - if alloc != 0 { - // Only fill the extended resource entry when it's non-zero. - allocatable[resource], requested[resource] = alloc, req + requested := make([]int64, len(r.resources)) + allocatable := make([]int64, len(r.resources)) + for i := range r.resources { + alloc, req := r.calculateResourceAllocatableRequest(nodeInfo, pod, v1.ResourceName(r.resources[i].Name)) + // Only fill the extended resource entry when it's non-zero. + if alloc == 0 { + continue } + allocatable[i] = alloc + requested[i] = req } score := r.scorer(requested, allocatable) @@ -137,12 +134,3 @@ func (r *resourceAllocationScorer) calculatePodResourceRequest(pod *v1.Pod, reso return podRequest } - -// resourcesToWeightMap make weightmap from resources spec -func resourcesToWeightMap(resources []config.ResourceSpec) resourceToWeightMap { - resourceToWeightMap := make(resourceToWeightMap) - for _, resource := range resources { - resourceToWeightMap[v1.ResourceName(resource.Name)] = resource.Weight - } - return resourceToWeightMap -}