Merge pull request #114390 from tangwz/improve_NodeResourcesFit_replace_small_maps_with_slices

Improve performance of NodeResourcesFit scoring

pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go

@@ -78,28 +78,25 @@ func NewBalancedAllocation(baArgs runtime.Object, h framework.Handle, fts featur
 		return nil, err
 	}
 
-	resToWeightMap := make(resourceToWeightMap)
-
-	for _, resource := range args.Resources {
-		resToWeightMap[v1.ResourceName(resource.Name)] = resource.Weight
-	}
-
 	return &BalancedAllocation{
 		handle: h,
 		resourceAllocationScorer: resourceAllocationScorer{
-			Name:                BalancedAllocationName,
+			Name:         BalancedAllocationName,
-			scorer:              balancedResourceScorer,
+			scorer:       balancedResourceScorer,
-			useRequested:        true,
+			useRequested: true,
-			resourceToWeightMap: resToWeightMap,
+			resources:    args.Resources,
 		},
 	}, nil
 }
 
-func balancedResourceScorer(requested, allocable resourceToValueMap) int64 {
+func balancedResourceScorer(requested, allocable []int64) int64 {
 	var resourceToFractions []float64
 	var totalFraction float64
-	for name, value := range requested {
+	for i := range requested {
-		fraction := float64(value) / float64(allocable[name])
+		if allocable[i] == 0 {
+			continue
+		}
+		fraction := float64(requested[i]) / float64(allocable[i])
 		if fraction > 1 {
 			fraction = 1
 		}

pkg/scheduler/framework/plugins/noderesources/fit.go

@@ -49,27 +49,27 @@ const (
 // nodeResourceStrategyTypeMap maps strategy to scorer implementation
 var nodeResourceStrategyTypeMap = map[config.ScoringStrategyType]scorer{
 	config.LeastAllocated: func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer {
-		resToWeightMap := resourcesToWeightMap(args.ScoringStrategy.Resources)
+		resources := args.ScoringStrategy.Resources
 		return &resourceAllocationScorer{
-			Name:                string(config.LeastAllocated),
+			Name:      string(config.LeastAllocated),
-			scorer:              leastResourceScorer(resToWeightMap),
+			scorer:    leastResourceScorer(resources),
-			resourceToWeightMap: resToWeightMap,
+			resources: resources,
 		}
 	},
 	config.MostAllocated: func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer {
-		resToWeightMap := resourcesToWeightMap(args.ScoringStrategy.Resources)
+		resources := args.ScoringStrategy.Resources
 		return &resourceAllocationScorer{
-			Name:                string(config.MostAllocated),
+			Name:      string(config.MostAllocated),
-			scorer:              mostResourceScorer(resToWeightMap),
+			scorer:    mostResourceScorer(resources),
-			resourceToWeightMap: resToWeightMap,
+			resources: resources,
 		}
 	},
 	config.RequestedToCapacityRatio: func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer {
-		resToWeightMap := resourcesToWeightMap(args.ScoringStrategy.Resources)
+		resources := args.ScoringStrategy.Resources
 		return &resourceAllocationScorer{
-			Name:                string(config.RequestedToCapacityRatio),
+			Name:      string(config.RequestedToCapacityRatio),
-			scorer:              requestedToCapacityRatioScorer(resToWeightMap, args.ScoringStrategy.RequestedToCapacityRatio.Shape),
+			scorer:    requestedToCapacityRatioScorer(resources, args.ScoringStrategy.RequestedToCapacityRatio.Shape),
-			resourceToWeightMap: resToWeightMap,
+			resources: resources,
 		}
 	},
 }

pkg/scheduler/framework/plugins/noderesources/fit_test.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/framework"
 	plfeature "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
+	plugintesting "k8s.io/kubernetes/pkg/scheduler/framework/plugins/testing"
 	"k8s.io/kubernetes/pkg/scheduler/framework/runtime"
 	"k8s.io/kubernetes/pkg/scheduler/internal/cache"
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
@@ -780,3 +781,115 @@ func TestFitScore(t *testing.T) {
 		})
 	}
 }
+
+var benchmarkResourceSet = []config.ResourceSpec{
+	{Name: string(v1.ResourceCPU), Weight: 1},
+	{Name: string(v1.ResourceMemory), Weight: 1},
+	{Name: string(v1.ResourcePods), Weight: 1},
+	{Name: string(v1.ResourceStorage), Weight: 1},
+	{Name: string(v1.ResourceEphemeralStorage), Weight: 1},
+	{Name: string(extendedResourceA), Weight: 1},
+	{Name: string(extendedResourceB), Weight: 1},
+	{Name: string(kubernetesIOResourceA), Weight: 1},
+	{Name: string(kubernetesIOResourceB), Weight: 1},
+	{Name: string(hugePageResourceA), Weight: 1},
+}
+
+func BenchmarkTestFitScore(b *testing.B) {
+	tests := []struct {
+		name                 string
+		nodeResourcesFitArgs config.NodeResourcesFitArgs
+	}{
+		{
+			name: "RequestedToCapacityRatio with defaultResources",
+			nodeResourcesFitArgs: config.NodeResourcesFitArgs{
+				ScoringStrategy: &config.ScoringStrategy{
+					Type:      config.RequestedToCapacityRatio,
+					Resources: defaultResources,
+					RequestedToCapacityRatio: &config.RequestedToCapacityRatioParam{
+						Shape: []config.UtilizationShapePoint{
+							{Utilization: 0, Score: 10},
+							{Utilization: 100, Score: 0},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "RequestedToCapacityRatio with 10 resources",
+			nodeResourcesFitArgs: config.NodeResourcesFitArgs{
+				ScoringStrategy: &config.ScoringStrategy{
+					Type:      config.RequestedToCapacityRatio,
+					Resources: benchmarkResourceSet,
+					RequestedToCapacityRatio: &config.RequestedToCapacityRatioParam{
+						Shape: []config.UtilizationShapePoint{
+							{Utilization: 0, Score: 10},
+							{Utilization: 100, Score: 0},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "MostAllocated with defaultResources",
+			nodeResourcesFitArgs: config.NodeResourcesFitArgs{
+				ScoringStrategy: &config.ScoringStrategy{
+					Type:      config.MostAllocated,
+					Resources: defaultResources,
+				},
+			},
+		},
+		{
+			name: "MostAllocated with 10 resources",
+			nodeResourcesFitArgs: config.NodeResourcesFitArgs{
+				ScoringStrategy: &config.ScoringStrategy{
+					Type:      config.MostAllocated,
+					Resources: benchmarkResourceSet,
+				},
+			},
+		},
+		{
+			name: "LeastAllocated with defaultResources",
+			nodeResourcesFitArgs: config.NodeResourcesFitArgs{
+				ScoringStrategy: &config.ScoringStrategy{
+					Type:      config.LeastAllocated,
+					Resources: defaultResources,
+				},
+			},
+		},
+		{
+			name: "LeastAllocated with 10 resources",
+			nodeResourcesFitArgs: config.NodeResourcesFitArgs{
+				ScoringStrategy: &config.ScoringStrategy{
+					Type:      config.LeastAllocated,
+					Resources: benchmarkResourceSet,
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		b.Run(test.name, func(b *testing.B) {
+			existingPods := []*v1.Pod{
+				st.MakePod().Node("node1").Req(map[v1.ResourceName]string{"cpu": "2000", "memory": "4000"}).Obj(),
+			}
+			nodes := []*v1.Node{
+				st.MakeNode().Name("node1").Capacity(map[v1.ResourceName]string{"cpu": "4000", "memory": "10000"}).Obj(),
+			}
+			state := framework.NewCycleState()
+			var nodeResourcesFunc = runtime.FactoryAdapter(plfeature.Features{}, NewFit)
+			pl := plugintesting.SetupPlugin(b, nodeResourcesFunc, &test.nodeResourcesFitArgs, cache.NewSnapshot(existingPods, nodes))
+			p := pl.(*Fit)
+
+			b.ResetTimer()
+
+			requestedPod := st.MakePod().Req(map[v1.ResourceName]string{"cpu": "1000", "memory": "2000"}).Obj()
+			for i := 0; i < b.N; i++ {
+				_, status := p.Score(context.Background(), state, requestedPod, nodes[0].Name)
+				if !status.IsSuccess() {
+					b.Errorf("unexpected status: %v", status)
+				}
+			}
+		})
+	}
+}

pkg/scheduler/framework/plugins/noderesources/least_allocated.go

@@ -17,6 +17,7 @@ limitations under the License.
 package noderesources
 
 import (
+	"k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/framework"
 )
 
@@ -26,12 +27,15 @@ import (
 //
 // Details:
 // (cpu((capacity-requested)*MaxNodeScore*cpuWeight/capacity) + memory((capacity-requested)*MaxNodeScore*memoryWeight/capacity) + ...)/weightSum
-func leastResourceScorer(resToWeightMap resourceToWeightMap) func(resourceToValueMap, resourceToValueMap) int64 {
+func leastResourceScorer(resources []config.ResourceSpec) func([]int64, []int64) int64 {
-	return func(requested, allocable resourceToValueMap) int64 {
+	return func(requested, allocable []int64) int64 {
 		var nodeScore, weightSum int64
-		for resource := range requested {
+		for i := range requested {
-			weight := resToWeightMap[resource]
+			if allocable[i] == 0 {
-			resourceScore := leastRequestedScore(requested[resource], allocable[resource])
+				continue
+			}
+			weight := resources[i].Weight
+			resourceScore := leastRequestedScore(requested[i], allocable[i])
 			nodeScore += resourceScore * weight
 			weightSum += weight
 		}

pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go

@@ -360,6 +360,27 @@ func TestLeastAllocatedScoringStrategy(t *testing.T) {
 			expectedScores: []framework.NodeScore{{Name: "node1", Score: 50}, {Name: "node2", Score: 60}},
 			resources:      extendedResourceSet,
 		},
+		{
+			// If the node doesn't have a resource
+			// CPU Score: ((6000 - 3000) * 100) / 6000 = 50
+			// Memory Score: ((10000 - 4000) * 100) / 10000 = 60
+			// Node1 Score: (50 * 1 + 60 * 1) / (1 + 1) = 55
+			// Node2 Score: (50 * 1 + 60 * 1) / (1 + 1) = 55
+			name: "if the node doesn't have a resource",
+			requestedPod: st.MakePod().Node("node1").
+				Req(map[v1.ResourceName]string{"cpu": "3000", "memory": "4000"}).
+				Obj(),
+			nodes: []*v1.Node{
+				st.MakeNode().Name("node1").Capacity(map[v1.ResourceName]string{"cpu": "6000", "memory": "10000"}).Obj(),
+				st.MakeNode().Name("node2").Capacity(map[v1.ResourceName]string{"cpu": "6000", "memory": "10000", v1.ResourceName(extendedRes): "4"}).Obj(),
+			},
+			expectedScores: []framework.NodeScore{{Name: "node1", Score: 55}, {Name: "node2", Score: 55}},
+			resources: []config.ResourceSpec{
+				{Name: extendedRes, Weight: 2},
+				{Name: string(v1.ResourceCPU), Weight: 1},
+				{Name: string(v1.ResourceMemory), Weight: 1},
+			},
+		},
 	}
 
 	for _, test := range tests {

pkg/scheduler/framework/plugins/noderesources/most_allocated.go

@@ -17,6 +17,7 @@ limitations under the License.
 package noderesources
 
 import (
+	"k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/framework"
 )
 
@@ -26,12 +27,15 @@ import (
 //
 // Details:
 // (cpu(MaxNodeScore * requested * cpuWeight / capacity) + memory(MaxNodeScore * requested * memoryWeight / capacity) + ...) / weightSum
-func mostResourceScorer(resToWeightMap resourceToWeightMap) func(requested, allocable resourceToValueMap) int64 {
+func mostResourceScorer(resources []config.ResourceSpec) func(requested, allocable []int64) int64 {
-	return func(requested, allocable resourceToValueMap) int64 {
+	return func(requested, allocable []int64) int64 {
 		var nodeScore, weightSum int64
-		for resource := range requested {
+		for i := range requested {
-			weight := resToWeightMap[resource]
+			if allocable[i] == 0 {
-			resourceScore := mostRequestedScore(requested[resource], allocable[resource])
+				continue
+			}
+			weight := resources[i].Weight
+			resourceScore := mostRequestedScore(requested[i], allocable[i])
 			nodeScore += resourceScore * weight
 			weightSum += weight
 		}

pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go

@@ -317,6 +317,27 @@ func TestMostAllocatedScoringStrategy(t *testing.T) {
 			existingPods:   nil,
 			expectedScores: []framework.NodeScore{{Name: "node1", Score: 50}, {Name: "node2", Score: 40}},
 		},
+		{
+			// If the node doesn't have a resource
+			// CPU Score: (3000 * 100) / 6000 = 50
+			// Memory Score: (4000 * 100) / 10000 = 40
+			// Node1 Score: (50 * 1 + 40 * 1) / (1 + 1) = 45
+			// Node2 Score: (50 * 1 + 40 * 1) / (1 + 1) = 45
+			name: "if the node doesn't have a resource",
+			requestedPod: st.MakePod().Node("node1").
+				Req(map[v1.ResourceName]string{"cpu": "3000", "memory": "4000"}).
+				Obj(),
+			nodes: []*v1.Node{
+				st.MakeNode().Name("node1").Capacity(map[v1.ResourceName]string{"cpu": "6000", "memory": "10000"}).Obj(),
+				st.MakeNode().Name("node2").Capacity(map[v1.ResourceName]string{"cpu": "6000", "memory": "10000", v1.ResourceName(extendedRes): "4"}).Obj(),
+			},
+			expectedScores: []framework.NodeScore{{Name: "node1", Score: 45}, {Name: "node2", Score: 45}},
+			resources: []config.ResourceSpec{
+				{Name: extendedRes, Weight: 2},
+				{Name: string(v1.ResourceCPU), Weight: 1},
+				{Name: string(v1.ResourceMemory), Weight: 1},
+			},
+		},
 	}
 
 	for _, test := range tests {

pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go

@@ -28,7 +28,7 @@ const maxUtilization = 100
 
 // buildRequestedToCapacityRatioScorerFunction allows users to apply bin packing
 // on core resources like CPU, Memory as well as extended resources like accelerators.
-func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resourceToWeightMap resourceToWeightMap) func(resourceToValueMap, resourceToValueMap) int64 {
+func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resources []config.ResourceSpec) func([]int64, []int64) int64 {
 	rawScoringFunction := helper.BuildBrokenLinearFunction(scoringFunctionShape)
 	resourceScoringFunction := func(requested, capacity int64) int64 {
 		if capacity == 0 || requested > capacity {
@@ -37,11 +37,14 @@ func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.Fun
 
 		return rawScoringFunction(requested * maxUtilization / capacity)
 	}
-	return func(requested, allocable resourceToValueMap) int64 {
+	return func(requested, allocable []int64) int64 {
 		var nodeScore, weightSum int64
-		for resource := range requested {
+		for i := range requested {
-			weight := resourceToWeightMap[resource]
+			if allocable[i] == 0 {
-			resourceScore := resourceScoringFunction(requested[resource], allocable[resource])
+				continue
+			}
+			weight := resources[i].Weight
+			resourceScore := resourceScoringFunction(requested[i], allocable[i])
 			if resourceScore > 0 {
 				nodeScore += resourceScore * weight
 				weightSum += weight
@@ -54,7 +57,7 @@ func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.Fun
 	}
 }
 
-func requestedToCapacityRatioScorer(weightMap resourceToWeightMap, shape []config.UtilizationShapePoint) func(resourceToValueMap, resourceToValueMap) int64 {
+func requestedToCapacityRatioScorer(resources []config.ResourceSpec, shape []config.UtilizationShapePoint) func([]int64, []int64) int64 {
 	shapes := make([]helper.FunctionShapePoint, 0, len(shape))
 	for _, point := range shape {
 		shapes = append(shapes, helper.FunctionShapePoint{
@@ -66,5 +69,5 @@ func requestedToCapacityRatioScorer(weightMap resourceToWeightMap, shape []confi
 		})
 	}
 
-	return buildRequestedToCapacityRatioScorerFunction(shapes, weightMap)
+	return buildRequestedToCapacityRatioScorerFunction(shapes, resources)
 }

pkg/scheduler/framework/plugins/noderesources/resource_allocation.go

@@ -24,9 +24,6 @@ import (
 	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
 )
 
-// resourceToWeightMap contains resource name and weight.
-type resourceToWeightMap map[v1.ResourceName]int64
-
 // scorer is decorator for resourceAllocationScorer
 type scorer func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer
 
@@ -35,14 +32,11 @@ type resourceAllocationScorer struct {
 	Name string
 	// used to decide whether to use Requested or NonZeroRequested for
 	// cpu and memory.
-	useRequested        bool
+	useRequested bool
-	scorer              func(requested, allocable resourceToValueMap) int64
+	scorer       func(requested, allocable []int64) int64
-	resourceToWeightMap resourceToWeightMap
+	resources    []config.ResourceSpec
 }
 
-// resourceToValueMap is keyed with resource name and valued with quantity.
-type resourceToValueMap map[v1.ResourceName]int64
-
 // score will use `scorer` function to calculate the score.
 func (r *resourceAllocationScorer) score(
 	pod *v1.Pod,
@@ -51,18 +45,21 @@ func (r *resourceAllocationScorer) score(
 	if node == nil {
 		return 0, framework.NewStatus(framework.Error, "node not found")
 	}
-	if len(r.resourceToWeightMap) == 0 {
+	// resources not set, nothing scheduled,
+	if len(r.resources) == 0 {
 		return 0, framework.NewStatus(framework.Error, "resources not found")
 	}
 
-	requested := make(resourceToValueMap)
+	requested := make([]int64, len(r.resources))
-	allocatable := make(resourceToValueMap)
+	allocatable := make([]int64, len(r.resources))
-	for resource := range r.resourceToWeightMap {
+	for i := range r.resources {
-		alloc, req := r.calculateResourceAllocatableRequest(nodeInfo, pod, resource)
+		alloc, req := r.calculateResourceAllocatableRequest(nodeInfo, pod, v1.ResourceName(r.resources[i].Name))
-		if alloc != 0 {
+		// Only fill the extended resource entry when it's non-zero.
-			// Only fill the extended resource entry when it's non-zero.
+		if alloc == 0 {
-			allocatable[resource], requested[resource] = alloc, req
+			continue
 		}
+		allocatable[i] = alloc
+		requested[i] = req
 	}
 
 	score := r.scorer(requested, allocatable)
@@ -137,12 +134,3 @@ func (r *resourceAllocationScorer) calculatePodResourceRequest(pod *v1.Pod, reso
 
 	return podRequest
 }
-
-// resourcesToWeightMap make weightmap from resources spec
-func resourcesToWeightMap(resources []config.ResourceSpec) resourceToWeightMap {
-	resourceToWeightMap := make(resourceToWeightMap)
-	for _, resource := range resources {
-		resourceToWeightMap[v1.ResourceName(resource.Name)] = resource.Weight
-	}
-	return resourceToWeightMap
-}