From dd6f4d3a16306c9cc8dbc6a210c0b7a38fa1eb68 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 3 Mar 2026 16:21:38 +0100 Subject: [PATCH] DRA scheduler: avoid panic during PreBind It can happen that a claim gets deallocated in parallel to adding a new pod to ReservedFor. Without binding conditions, that was caught by the apiserver validation. With binding conditions, the code which checks and sets AllocationTimestamp panics with a nil pointer access. This has been observed in the TestDRA/all/ShareResourceClaimSequentially integration test, but couldn't be reproduced locally: E0303 07:43:20.158261 39037 panic.go:262] "Observed a panic" panic="runtime error: invalid memory address or nil pointer dereference" panicGoValue="\"invalid memory address or nil pointer dereference\"" stacktrace=< goroutine 554266 [running]: k8s.io/apimachinery/pkg/util/runtime.logPanic({0x69ce9f0, 0xc017bc00f0}, {0x59381a0, 0x91c6570}) /home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go:132 +0xbc k8s.io/apimachinery/pkg/util/runtime.handleCrash({0x69ce8d8, 0x93d87a0}, {0x59381a0, 0x91c6570}, {0xc020506f00, 0x0, 0x200?}) /home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go:107 +0x116 k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc00685ddc0?}) /home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go:64 +0x17b panic({0x59381a0?, 0x91c6570?}) /usr/local/go/src/runtime/panic.go:783 +0x132 k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).bindClaim.func2() /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go:1247 +0x730 k8s.io/client-go/util/retry.OnError.func1() /home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/retry/util.go:51 +0x30 k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection(0x9ebcca?) /home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/wait/wait.go:150 +0x3e k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff({0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x2, 0x0}, 0xc020507410) /home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/wait/backoff.go:477 +0x5a k8s.io/client-go/util/retry.OnError({0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc02c2d5380?, 0x4?) /home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/retry/util.go:50 +0x96 k8s.io/client-go/util/retry.RetryOnConflict(...) /home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/retry/util.go:104 k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).bindClaim(0xc0024adb20, {0x69cea28, 0xc0061acbe0}, 0xc011876800, 0x0, 0xc025163408, {0xc021909d80, 0x8}) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go:1207 +0x845 k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).PreBind-range1(...) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go:1073 k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).PreBind.(*claimStore).all.func2(...) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/claims.go:72 k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).PreBind(0xc0024adb20, {0x69cea28, 0xc0061acbe0}, {0x69fc840?, 0xc0286f2540?}, 0xc025163408, {0xc021909d80, 0x8}) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go:1071 +0x246 k8s.io/kubernetes/pkg/scheduler/framework/runtime.(*frameworkImpl).runPreBindPlugin(0xc009628dc8, {0x69cea28, 0xc0061acbe0}, {0x7eabfddddb30, 0xc0024adb20}, {0x69fc840, 0xc0286f2540}, 0xc025163408, {0xc021909d80, 0x8}) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/runtime/framework.go:1532 +0x2e2 k8s.io/kubernetes/pkg/scheduler/framework/runtime.(*frameworkImpl).RunPreBindPlugins.func2({0x7eabfddddb30, 0xc0024adb20}) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/runtime/framework.go:1461 +0x1cf k8s.io/kubernetes/pkg/scheduler/framework/runtime.(*frameworkImpl).RunPreBindPlugins(0xc009628dc8, {0x69cea28, 0xc0061ac690}, {0x69fc840, 0xc0286f2540}, 0xc025163408, {0xc021909d80, 0x8}) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/runtime/framework.go:1484 +0x623 k8s.io/kubernetes/pkg/scheduler.(*Scheduler).bindingCycle(0xc02a6a7500, {0x69cea28, 0xc00f22a690}, {0x69fc840, 0xc0286f2540}, {0x6a32e60, 0xc009628dc8}, {{0xc021909d80, 0x8}, 0x8, ...}, ...) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/schedule_one.go:457 +0x72a k8s.io/kubernetes/pkg/scheduler.(*Scheduler).runBindingCycle(0xc02a6a7500, {0x69ce9f0?, 0xc0027e5470?}, {0x69fc840, 0xc0286f2540}, {0x6a32e60, 0xc009628dc8}, {{0xc021909d80, 0x8}, 0x8, ...}, ...) /home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/schedule_one.go:164 +0x1e8 --- .../plugins/dynamicresources/dynamicresources.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go index a85fe3e0291..1a858c26eed 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go @@ -1244,8 +1244,17 @@ func (pl *DynamicResources) bindClaim(ctx context.Context, state *stateData, ind // preconditions. The apiserver will tell us with a // non-conflict error if this isn't possible. claim.Status.ReservedFor = append(claim.Status.ReservedFor, resourceapi.ResourceClaimConsumerReference{Resource: "pods", Name: pod.Name, UID: pod.UID}) - if pl.fts.EnableDRADeviceBindingConditions && pl.fts.EnableDRAResourceClaimDeviceStatus && claim.Status.Allocation.AllocationTimestamp == nil { - claim.Status.Allocation.AllocationTimestamp = &metav1.Time{Time: time.Now()} + if pl.fts.EnableDRADeviceBindingConditions && + pl.fts.EnableDRAResourceClaimDeviceStatus { + // Once the feature gate checks get removed, this code can be moved into an else branch + // for `if allocation != nil` above. For now it is kept here to avoid changing normal + // behavior. + if claim.Status.Allocation == nil { + return fmt.Errorf("claim %s got deallocated elsewhere in the meantime", klog.KObj(claim)) + } + if claim.Status.Allocation.AllocationTimestamp == nil { + claim.Status.Allocation.AllocationTimestamp = &metav1.Time{Time: time.Now()} + } } updatedClaim, err := pl.clientset.ResourceV1().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{}) if err != nil {