From f63702de0f958b6364a577291fb53022d431eb08 Mon Sep 17 00:00:00 2001 From: Haosdent Huang Date: Thu, 17 Apr 2025 18:01:03 +0800 Subject: [PATCH] scheduler: return UnschedulableAndUnresolvable when node capacity is insufficient Currently, the NodeResourcesFit plugin always returns Unschedulable when a pod's resource requests exceed a node's available resources. However, when a pod's requests exceed the node's total allocatable, preemption cannot help since even an empty node would not have enough resources. This change modifies the NodeResourcesFit plugin to return UnschedulableAndUnresolvable when a pod's resource requests exceed the node's total allocatable. This helps optimize the scheduling process in large clusters by: 1. Reducing the number of candidate nodes that need to be considered for preemption 2. Providing clearer feedback about unresolvable resource constraints 3. Improving scheduling performance by avoiding unnecessary preemption calculations The change is particularly beneficial in heterogeneous clusters where node sizes vary significantly, as it helps quickly identify nodes that are fundamentally too small for certain pods. Fixes https://github.com/kubernetes/kubernetes/issues/131310 Co-authored-by: Kensei Nakada --- .../framework/plugins/noderesources/fit.go | 15 ++++- .../plugins/noderesources/fit_test.go | 57 ++++++++++--------- pkg/scheduler/schedule_one_test.go | 2 +- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/pkg/scheduler/framework/plugins/noderesources/fit.go b/pkg/scheduler/framework/plugins/noderesources/fit.go index d94554ad6d7..a7dba32864e 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit.go @@ -463,10 +463,16 @@ func (f *Fit) Filter(ctx context.Context, cycleState *framework.CycleState, pod if len(insufficientResources) != 0 { // We will keep all failure reasons. failureReasons := make([]string, 0, len(insufficientResources)) + statusCode := framework.Unschedulable for i := range insufficientResources { failureReasons = append(failureReasons, insufficientResources[i].Reason) + + if insufficientResources[i].Unresolvable { + statusCode = framework.UnschedulableAndUnresolvable + } } - return framework.NewStatus(framework.Unschedulable, failureReasons...) + + return framework.NewStatus(statusCode, failureReasons...) } return nil } @@ -489,6 +495,9 @@ type InsufficientResource struct { Requested int64 Used int64 Capacity int64 + // Unresolvable indicates whether this node could be schedulable for the pod by the preemption, + // which is determined by comparing the node's size and the pod's request. + Unresolvable bool } // Fits checks if node have enough resources to host the pod. @@ -524,6 +533,7 @@ func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignor Requested: podRequest.MilliCPU, Used: nodeInfo.Requested.MilliCPU, Capacity: nodeInfo.Allocatable.MilliCPU, + Unresolvable: podRequest.MilliCPU > nodeInfo.Allocatable.MilliCPU, }) } if podRequest.Memory > 0 && podRequest.Memory > (nodeInfo.Allocatable.Memory-nodeInfo.Requested.Memory) { @@ -533,6 +543,7 @@ func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignor Requested: podRequest.Memory, Used: nodeInfo.Requested.Memory, Capacity: nodeInfo.Allocatable.Memory, + Unresolvable: podRequest.Memory > nodeInfo.Allocatable.Memory, }) } if podRequest.EphemeralStorage > 0 && @@ -543,6 +554,7 @@ func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignor Requested: podRequest.EphemeralStorage, Used: nodeInfo.Requested.EphemeralStorage, Capacity: nodeInfo.Allocatable.EphemeralStorage, + Unresolvable: podRequest.EphemeralStorage > nodeInfo.Allocatable.EphemeralStorage, }) } @@ -571,6 +583,7 @@ func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignor Requested: podRequest.ScalarResources[rName], Used: nodeInfo.Requested.ScalarResources[rName], Capacity: nodeInfo.Allocatable.ScalarResources[rName], + Unresolvable: rQuant > nodeInfo.Allocatable.ScalarResources[rName], }) } } diff --git a/pkg/scheduler/framework/plugins/noderesources/fit_test.go b/pkg/scheduler/framework/plugins/noderesources/fit_test.go index ab46344d906..df0ad7e0b9d 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit_test.go @@ -268,9 +268,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0, ScalarResources: map[v1.ResourceName]int64{extendedResourceA: 0}})), name: "extended resource capacity enforced", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(extendedResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(extendedResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: extendedResourceA, Reason: getErrReason(extendedResourceA), Requested: 10, Used: 0, Capacity: 5}, + {ResourceName: extendedResourceA, Reason: getErrReason(extendedResourceA), Requested: 10, Used: 0, Capacity: 5, Unresolvable: true}, }, }, { @@ -279,9 +279,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0, ScalarResources: map[v1.ResourceName]int64{extendedResourceA: 0}})), name: "extended resource capacity enforced for init container", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(extendedResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(extendedResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: extendedResourceA, Reason: getErrReason(extendedResourceA), Requested: 10, Used: 0, Capacity: 5}, + {ResourceName: extendedResourceA, Reason: getErrReason(extendedResourceA), Requested: 10, Used: 0, Capacity: 5, Unresolvable: true}, }, }, { @@ -313,9 +313,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0, ScalarResources: map[v1.ResourceName]int64{extendedResourceA: 2}})), name: "extended resource allocatable enforced for multiple containers", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(extendedResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(extendedResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: extendedResourceA, Reason: getErrReason(extendedResourceA), Requested: 6, Used: 2, Capacity: 5}, + {ResourceName: extendedResourceA, Reason: getErrReason(extendedResourceA), Requested: 6, Used: 2, Capacity: 5, Unresolvable: true}, }, }, { @@ -334,9 +334,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0, ScalarResources: map[v1.ResourceName]int64{extendedResourceA: 2}})), name: "extended resource allocatable enforced for multiple init containers", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(extendedResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(extendedResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: extendedResourceA, Reason: getErrReason(extendedResourceA), Requested: 6, Used: 2, Capacity: 5}, + {ResourceName: extendedResourceA, Reason: getErrReason(extendedResourceA), Requested: 6, Used: 2, Capacity: 5, Unresolvable: true}, }, }, { @@ -345,9 +345,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0})), name: "extended resource allocatable enforced for unknown resource", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(extendedResourceB)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(extendedResourceB)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: extendedResourceB, Reason: getErrReason(extendedResourceB), Requested: 1, Used: 0, Capacity: 0}, + {ResourceName: extendedResourceB, Reason: getErrReason(extendedResourceB), Requested: 1, Used: 0, Capacity: 0, Unresolvable: true}, }, }, { @@ -356,9 +356,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0})), name: "extended resource allocatable enforced for unknown resource for init container", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(extendedResourceB)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(extendedResourceB)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: extendedResourceB, Reason: getErrReason(extendedResourceB), Requested: 1, Used: 0, Capacity: 0}, + {ResourceName: extendedResourceB, Reason: getErrReason(extendedResourceB), Requested: 1, Used: 0, Capacity: 0, Unresolvable: true}, }, }, { @@ -367,9 +367,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0})), name: "kubernetes.io resource capacity enforced", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(kubernetesIOResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(kubernetesIOResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: kubernetesIOResourceA, Reason: getErrReason(kubernetesIOResourceA), Requested: 10, Used: 0, Capacity: 0}, + {ResourceName: kubernetesIOResourceA, Reason: getErrReason(kubernetesIOResourceA), Requested: 10, Used: 0, Capacity: 0, Unresolvable: true}, }, }, { @@ -378,9 +378,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0})), name: "kubernetes.io resource capacity enforced for init container", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(kubernetesIOResourceB)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(kubernetesIOResourceB)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: kubernetesIOResourceB, Reason: getErrReason(kubernetesIOResourceB), Requested: 10, Used: 0, Capacity: 0}, + {ResourceName: kubernetesIOResourceB, Reason: getErrReason(kubernetesIOResourceB), Requested: 10, Used: 0, Capacity: 0, Unresolvable: true}, }, }, { @@ -389,9 +389,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0, ScalarResources: map[v1.ResourceName]int64{hugePageResourceA: 0}})), name: "hugepages resource capacity enforced", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(hugePageResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(hugePageResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 10, Used: 0, Capacity: 5}, + {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 10, Used: 0, Capacity: 5, Unresolvable: true}, }, }, { @@ -400,9 +400,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0, ScalarResources: map[v1.ResourceName]int64{hugePageResourceA: 0}})), name: "hugepages resource capacity enforced for init container", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(hugePageResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(hugePageResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 10, Used: 0, Capacity: 5}, + {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 10, Used: 0, Capacity: 5, Unresolvable: true}, }, }, { @@ -412,9 +412,9 @@ func TestEnoughRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0, ScalarResources: map[v1.ResourceName]int64{hugePageResourceA: 2}})), name: "hugepages resource allocatable enforced for multiple containers", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(hugePageResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(hugePageResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 6, Used: 2, Capacity: 5}, + {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 6, Used: 2, Capacity: 5, Unresolvable: true}, }, }, { @@ -462,14 +462,15 @@ func TestEnoughRequests(t *testing.T) { IgnoredResourceGroups: []string{"example.com"}, }, name: "skip checking ignored extended resource via resource groups", - wantStatus: framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Insufficient %v", kubernetesIOResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(kubernetesIOResourceA)), wantInsufficientResources: []InsufficientResource{ { ResourceName: kubernetesIOResourceA, - Reason: fmt.Sprintf("Insufficient %v", kubernetesIOResourceA), + Reason: getErrReason(kubernetesIOResourceA), Requested: 1, Used: 0, Capacity: 0, + Unresolvable: true, }, }, }, @@ -577,9 +578,9 @@ func TestEnoughRequests(t *testing.T) { ), nodeInfo: framework.NewNodeInfo(), name: "pod-level hugepages resource not fit", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(hugePageResourceA)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(hugePageResourceA)), wantInsufficientResources: []InsufficientResource{ - {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 10, Used: 0, Capacity: 5}, + {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 10, Used: 0, Capacity: 5, Unresolvable: true}, }, }, { @@ -736,7 +737,7 @@ func TestStorageRequests(t *testing.T) { nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 2, Memory: 2})), name: "storage ephemeral local storage request exceeds allocatable", - wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(v1.ResourceEphemeralStorage)), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(v1.ResourceEphemeralStorage)), }, { pod: newResourceInitPod(newResourcePod(framework.Resource{EphemeralStorage: 5})), @@ -857,7 +858,7 @@ func TestRestartableInitContainers(t *testing.T) { &v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(1, resource.DecimalSI)}, &v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(2, resource.DecimalSI)}, ), - wantFilterStatus: framework.NewStatus(framework.Unschedulable, "Insufficient cpu"), + wantFilterStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, getErrReason(v1.ResourceCPU)), }, } diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go index 1d9943de3af..c2e84410dd5 100644 --- a/pkg/scheduler/schedule_one_test.go +++ b/pkg/scheduler/schedule_one_test.go @@ -1477,7 +1477,7 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { failedNodeStatues := framework.NewDefaultNodeToStatus() for _, node := range nodes { failedNodeStatues.Set(node.Name, framework.NewStatus( - framework.Unschedulable, + framework.UnschedulableAndUnresolvable, fmt.Sprintf("Insufficient %v", v1.ResourceCPU), fmt.Sprintf("Insufficient %v", v1.ResourceMemory), ).WithPlugin(noderesources.Name))