From 0680960e5e8da5a0ef97c8baf81c1a9b01931e2c Mon Sep 17 00:00:00 2001 From: Hamza <12420351+0xMH@users.noreply.github.com> Date: Wed, 11 Mar 2026 02:20:30 +0100 Subject: [PATCH 1/4] Return Error from DRA filter on timeout to enable automatic retry --- .../framework/plugins/dynamicresources/dynamicresources.go | 5 ++++- .../plugins/dynamicresources/dynamicresources_test.go | 7 +++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go index 46db7e45c09..80f9a1a37bf 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go @@ -721,7 +721,10 @@ func (pl *DynamicResources) Filter(ctx context.Context, cs fwk.CycleState, pod * a, err := state.allocator.Allocate(allocCtx, node, claimsToAllocate) switch { case errors.Is(err, context.DeadlineExceeded): - return statusUnschedulable(logger, "timed out trying to allocate devices", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaims", klog.KObjSlice(claimsToAllocate)) + // Timeouts are transient, not a property of the node. Return Error + // so the pod retries via backoff instead of sitting in the + // unschedulable queue waiting for a cluster event. + return statusError(logger, fmt.Errorf("timed out trying to allocate devices"), "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaims", klog.KObjSlice(claimsToAllocate)) case errors.Is(err, structured.ErrFailedAllocationOnNode): // Not a fatal error, allocation on other nodes may proceed. // The error is only surfaced if allocation fails on all nodes. diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go index 222fbbced21..d3d140e0009 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go @@ -2114,12 +2114,11 @@ func testPlugin(tCtx ktesting.TContext) { want: want{ filter: perNodeResult{ workerNode.Name: { - status: fwk.NewStatus(fwk.UnschedulableAndUnresolvable, `timed out trying to allocate devices`), + // Timeouts return Error so the pod retries via backoff. + status: fwk.AsStatus(fmt.Errorf("timed out trying to allocate devices")), }, }, - postfilter: result{ - status: fwk.NewStatus(fwk.Unschedulable, `still not schedulable`), - }, + // No postfilter: Error aborts scheduling immediately. }, // Skipping this test case on Windows as a 1ns timeout is not guaranteed to // expire immediately on Windows due to its coarser timer granularity - From 3ea232f22025758221adde4aac8b017a08eeb8c3 Mon Sep 17 00:00:00 2001 From: Hamza <12420351+0xMH@users.noreply.github.com> Date: Tue, 17 Mar 2026 11:29:37 +0100 Subject: [PATCH 2/4] DRA: adapt FilterTimeout test after upstream dra_test.go split --- test/integration/dra/core.go | 5 +++-- test/integration/dra/helpers.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/test/integration/dra/core.go b/test/integration/dra/core.go index 6d4bacdf849..876f7e2fac7 100644 --- a/test/integration/dra/core.go +++ b/test/integration/dra/core.go @@ -133,10 +133,11 @@ profiles: args: filterTimeout: 10ms `) - expectPodUnschedulable(tCtx, pod, "timed out trying to allocate devices") + expectPodSchedulerError(tCtx, pod, "timed out trying to allocate devices") // Update one slice such that allocation succeeds. - // The scheduler must retry and should succeed now. + // The scheduler retries automatically (timeouts go through + // backoff queue, not unschedulable pool) and should succeed now. createdOtherSlice.Spec.Devices = append(createdOtherSlice.Spec.Devices, resourceapi.Device{ Name: fmt.Sprintf("dev-%d", devicesPerSlice), }) diff --git a/test/integration/dra/helpers.go b/test/integration/dra/helpers.go index b52e6c10add..f3f24126dd2 100644 --- a/test/integration/dra/helpers.go +++ b/test/integration/dra/helpers.go @@ -272,6 +272,20 @@ func waitForClaimAllocatedToDevice(tCtx ktesting.TContext, namespace, claimName ) } +func expectPodSchedulerError(tCtx ktesting.TContext, pod *v1.Pod, reason string) { + tCtx.Helper() + tCtx.ExpectNoError(e2epod.WaitForPodCondition(tCtx, tCtx.Client(), pod.Namespace, pod.Name, v1.PodReasonSchedulerError, time.Minute, func(pod *v1.Pod) (bool, error) { + if pod.Status.Phase == v1.PodPending { + for _, cond := range pod.Status.Conditions { + if cond.Type == v1.PodScheduled && cond.Status == v1.ConditionFalse && cond.Reason == v1.PodReasonSchedulerError && strings.Contains(cond.Message, reason) { + return true, nil + } + } + } + return false, nil + }), fmt.Sprintf("expected pod to have scheduler error because %q", reason)) +} + func expectPodUnschedulable(tCtx ktesting.TContext, pod *v1.Pod, reason string) { tCtx.Helper() tCtx.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(tCtx, tCtx.Client(), pod.Name, pod.Namespace), fmt.Sprintf("expected pod to be unschedulable because %q", reason)) From 77403e5940e6d8ea91b38e76818a1bcb55ab7c13 Mon Sep 17 00:00:00 2001 From: Hamza <12420351+0xMH@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:53:22 +0100 Subject: [PATCH 3/4] DRA: fix FilterTimeout/enabled test to clear worker-0 devices --- test/integration/dra/core.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/integration/dra/core.go b/test/integration/dra/core.go index 876f7e2fac7..98ac54d4ac8 100644 --- a/test/integration/dra/core.go +++ b/test/integration/dra/core.go @@ -103,7 +103,7 @@ func testFilterTimeout(tCtx ktesting.TContext, devicesPerSlice int) { deviceNames[i] = fmt.Sprintf("dev-%d", i) } slice := st.MakeResourceSlice("worker-0", driverName).Devices(deviceNames...) - createSlice(tCtx, slice.Obj()) + createdSlice := createSlice(tCtx, slice.Obj()) otherSlice := st.MakeResourceSlice("worker-1", driverName).Devices(deviceNames...) createdOtherSlice := createSlice(tCtx, otherSlice.Obj()) claim := claim.DeepCopy() @@ -135,13 +135,21 @@ profiles: `) expectPodSchedulerError(tCtx, pod, "timed out trying to allocate devices") - // Update one slice such that allocation succeeds. + // Clear worker-0's devices so the allocator completes quickly + // on that node. With Error status (from timeouts), the scheduler + // aborts the entire cycle, so no node with a slow allocation + // can remain. + createdSlice.Spec.Devices = nil + _, err := tCtx.Client().ResourceV1().ResourceSlices().Update(tCtx, createdSlice, metav1.UpdateOptions{}) + tCtx.ExpectNoError(err, "clear worker-0's ResourceSlice") + + // Update worker-1 so allocation succeeds. // The scheduler retries automatically (timeouts go through // backoff queue, not unschedulable pool) and should succeed now. createdOtherSlice.Spec.Devices = append(createdOtherSlice.Spec.Devices, resourceapi.Device{ Name: fmt.Sprintf("dev-%d", devicesPerSlice), }) - _, err := tCtx.Client().ResourceV1().ResourceSlices().Update(tCtx, createdOtherSlice, metav1.UpdateOptions{}) + _, err = tCtx.Client().ResourceV1().ResourceSlices().Update(tCtx, createdOtherSlice, metav1.UpdateOptions{}) tCtx.ExpectNoError(err, "update worker-1's ResourceSlice") tCtx.ExpectNoError(e2epod.WaitForPodScheduled(tCtx, tCtx.Client(), namespace, pod.Name)) }) From 67deec3bfae701dc5ddcea6c79be4b8c1abd43fc Mon Sep 17 00:00:00 2001 From: Hamza <12420351+0xMH@users.noreply.github.com> Date: Wed, 18 Mar 2026 00:55:58 +0100 Subject: [PATCH 4/4] DRA: include node name in timeout error and rework FilterTimeout test setup --- .../dynamicresources/dynamicresources.go | 7 ++- .../dynamicresources/dynamicresources_test.go | 2 +- test/integration/dra/core.go | 46 ++++++++++--------- test/integration/dra/dra.go | 4 +- test/integration/dra/helpers.go | 13 ------ 5 files changed, 32 insertions(+), 40 deletions(-) diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go index 80f9a1a37bf..bdaaa0bb569 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go @@ -721,10 +721,13 @@ func (pl *DynamicResources) Filter(ctx context.Context, cs fwk.CycleState, pod * a, err := state.allocator.Allocate(allocCtx, node, claimsToAllocate) switch { case errors.Is(err, context.DeadlineExceeded): - // Timeouts are transient, not a property of the node. Return Error + // Timeouts are potentially transient. Return Error // so the pod retries via backoff instead of sitting in the // unschedulable queue waiting for a cluster event. - return statusError(logger, fmt.Errorf("timed out trying to allocate devices"), "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaims", klog.KObjSlice(claimsToAllocate)) + // + // The timeout might be caused by ResourceSlices for the node, + // so including the node name may help with diagnosing the failure. + return statusError(logger, fmt.Errorf("node %s: timed out trying to allocate devices", node.Name), "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaims", klog.KObjSlice(claimsToAllocate)) case errors.Is(err, structured.ErrFailedAllocationOnNode): // Not a fatal error, allocation on other nodes may proceed. // The error is only surfaced if allocation fails on all nodes. diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go index d3d140e0009..79a00ae0994 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go @@ -2115,7 +2115,7 @@ func testPlugin(tCtx ktesting.TContext) { filter: perNodeResult{ workerNode.Name: { // Timeouts return Error so the pod retries via backoff. - status: fwk.AsStatus(fmt.Errorf("timed out trying to allocate devices")), + status: fwk.AsStatus(fmt.Errorf("node %s: timed out trying to allocate devices", workerNode.Name)), }, }, // No postfilter: Error aborts scheduling immediately. diff --git a/test/integration/dra/core.go b/test/integration/dra/core.go index 98ac54d4ac8..23a48b7aeb8 100644 --- a/test/integration/dra/core.go +++ b/test/integration/dra/core.go @@ -95,23 +95,30 @@ func testConvert(tCtx ktesting.TContext) { // testFilterTimeout covers the scheduler plugin's filter timeout configuration and behavior. // // It runs the scheduler with non-standard settings and thus cannot run in parallel. -func testFilterTimeout(tCtx ktesting.TContext, devicesPerSlice int) { +func testFilterTimeout(tCtx ktesting.TContext, requestDeviceCount int) { namespace := createTestNamespace(tCtx, nil) class, driverName := createTestClass(tCtx, namespace) - deviceNames := make([]string, devicesPerSlice) - for i := range devicesPerSlice { + deviceNames := make([]string, requestDeviceCount) + for i := range requestDeviceCount { deviceNames[i] = fmt.Sprintf("dev-%d", i) } slice := st.MakeResourceSlice("worker-0", driverName).Devices(deviceNames...) - createdSlice := createSlice(tCtx, slice.Obj()) - otherSlice := st.MakeResourceSlice("worker-1", driverName).Devices(deviceNames...) + createSlice(tCtx, slice.Obj()) + otherSlice := st.MakeResourceSlice("worker-1", driverName).Devices(deviceNames[:requestDeviceCount-1]...) createdOtherSlice := createSlice(tCtx, otherSlice.Obj()) - claim := claim.DeepCopy() - claim.Spec.Devices.Requests[0].Exactly.Count = int64(devicesPerSlice + 1) // Impossible to allocate. - claim = createClaim(tCtx, namespace, "", class, claim) + + // Impossible to allocate on worker-1: not enough devices, but allocation is too + // dumb to notice that upfront and keeps trying until it times out. + // On worker-0 we can allocate, but don't schedule because of the timeout on worker-1. + newClaim := func(suffix string) *resourceapi.ResourceClaim { + c := claim.DeepCopy() + c.Spec.Devices.Requests[0].Exactly.Count = int64(requestDeviceCount) + return createClaim(tCtx, namespace, suffix, class, c) + } runSubTest(tCtx, "disabled", func(tCtx ktesting.TContext) { - pod := createPod(tCtx, namespace, "", podWithClaimName, claim) + cl := newClaim("-disabled") + pod := createPod(tCtx, namespace, "-disabled", podWithClaimName, cl) startSchedulerWithConfig(tCtx, ` profiles: - schedulerName: default-scheduler @@ -120,11 +127,14 @@ profiles: args: filterTimeout: 0s `) - expectPodUnschedulable(tCtx, pod, "cannot allocate all claims") + // Without a timeout, the allocator runs to completion on both nodes. + // worker-0 has enough devices and succeeds, so the pod gets scheduled. + tCtx.ExpectNoError(e2epod.WaitForPodScheduled(tCtx, tCtx.Client(), namespace, pod.Name)) }) runSubTest(tCtx, "enabled", func(tCtx ktesting.TContext) { - pod := createPod(tCtx, namespace, "", podWithClaimName, claim) + cl := newClaim("-enabled") + pod := createPod(tCtx, namespace, "-enabled", podWithClaimName, cl) startSchedulerWithConfig(tCtx, ` profiles: - schedulerName: default-scheduler @@ -135,21 +145,13 @@ profiles: `) expectPodSchedulerError(tCtx, pod, "timed out trying to allocate devices") - // Clear worker-0's devices so the allocator completes quickly - // on that node. With Error status (from timeouts), the scheduler - // aborts the entire cycle, so no node with a slow allocation - // can remain. - createdSlice.Spec.Devices = nil - _, err := tCtx.Client().ResourceV1().ResourceSlices().Update(tCtx, createdSlice, metav1.UpdateOptions{}) - tCtx.ExpectNoError(err, "clear worker-0's ResourceSlice") - - // Update worker-1 so allocation succeeds. + // Update the smaller slice such that allocation also succeeds. // The scheduler retries automatically (timeouts go through // backoff queue, not unschedulable pool) and should succeed now. createdOtherSlice.Spec.Devices = append(createdOtherSlice.Spec.Devices, resourceapi.Device{ - Name: fmt.Sprintf("dev-%d", devicesPerSlice), + Name: deviceNames[requestDeviceCount-1], }) - _, err = tCtx.Client().ResourceV1().ResourceSlices().Update(tCtx, createdOtherSlice, metav1.UpdateOptions{}) + _, err := tCtx.Client().ResourceV1().ResourceSlices().Update(tCtx, createdOtherSlice, metav1.UpdateOptions{}) tCtx.ExpectNoError(err, "update worker-1's ResourceSlice") tCtx.ExpectNoError(e2epod.WaitForPodScheduled(tCtx, tCtx.Client(), namespace, pod.Name)) }) diff --git a/test/integration/dra/dra.go b/test/integration/dra/dra.go index 35ce2cc6daf..0fb153a387b 100644 --- a/test/integration/dra/dra.go +++ b/test/integration/dra/dra.go @@ -133,7 +133,7 @@ func run(tCtx ktesting.TContext, whatRE string) { runSubTest(tCtx, "EvictClusterWithSlices", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useNoRule) }) // Number of devices per slice is chosen so that Filter takes a few seconds: // without a timeout, the test doesn't run too long, but long enough that a short timeout triggers. - runSubTest(tCtx, "FilterTimeout", func(tCtx ktesting.TContext) { testFilterTimeout(tCtx, 20) }) + runSubTest(tCtx, "FilterTimeout", func(tCtx ktesting.TContext) { testFilterTimeout(tCtx, 21) }) runSubTest(tCtx, "UsesAllResources", testUsesAllResources) }, }, @@ -243,7 +243,7 @@ func run(tCtx ktesting.TContext, whatRE string) { // Number of devices per slice is chosen so that Filter takes a few seconds: The allocator // in the experimental channel has an improvement that requires a higher number here than // in the incubating and stable channels. - runSubTest(tCtx, "FilterTimeout", func(tCtx ktesting.TContext) { testFilterTimeout(tCtx, 20) }) + runSubTest(tCtx, "FilterTimeout", func(tCtx ktesting.TContext) { testFilterTimeout(tCtx, 21) }) runSubTest(tCtx, "ShareResourceClaimSequentially", testShareResourceClaimSequentially) runSubTest(tCtx, "UsesAllResources", testUsesAllResources) }, diff --git a/test/integration/dra/helpers.go b/test/integration/dra/helpers.go index f3f24126dd2..96f6967fe33 100644 --- a/test/integration/dra/helpers.go +++ b/test/integration/dra/helpers.go @@ -286,19 +286,6 @@ func expectPodSchedulerError(tCtx ktesting.TContext, pod *v1.Pod, reason string) }), fmt.Sprintf("expected pod to have scheduler error because %q", reason)) } -func expectPodUnschedulable(tCtx ktesting.TContext, pod *v1.Pod, reason string) { - tCtx.Helper() - tCtx.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(tCtx, tCtx.Client(), pod.Name, pod.Namespace), fmt.Sprintf("expected pod to be unschedulable because %q", reason)) - pod, err := tCtx.Client().CoreV1().Pods(pod.Namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err) - gomega.NewWithT(tCtx).Expect(pod).To(gomega.HaveField("Status.Conditions", gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "Type": gomega.Equal(v1.PodScheduled), - "Status": gomega.Equal(v1.ConditionFalse), - "Reason": gomega.Equal(v1.PodReasonUnschedulable), - "Message": gomega.ContainSubstring(reason), - })))) -} - type nodeInfo struct { name string driverName string