github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-07-17 20:00:07 +00:00
Code Issues Projects Releases Wiki Activity

DRA scheduler: avoid panic during PreBind

Browse Source 
It can happen that a claim gets deallocated in parallel to adding a new pod to
ReservedFor. Without binding conditions, that was caught by the apiserver
validation. With binding conditions, the code which checks and sets
AllocationTimestamp panics with a nil pointer access.

This has been observed in the TestDRA/all/ShareResourceClaimSequentially
integration test, but couldn't be reproduced locally:

    E0303 07:43:20.158261   39037 panic.go:262] "Observed a panic" panic="runtime error: invalid memory address or nil pointer dereference" panicGoValue="\"invalid memory address or nil pointer dereference\"" stacktrace=<
    	goroutine 554266 [running]:
    	k8s.io/apimachinery/pkg/util/runtime.logPanic({0x69ce9f0, 0xc017bc00f0}, {0x59381a0, 0x91c6570})
    		/home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go:132 +0xbc
    	k8s.io/apimachinery/pkg/util/runtime.handleCrash({0x69ce8d8, 0x93d87a0}, {0x59381a0, 0x91c6570}, {0xc020506f00, 0x0, 0x200?})
    		/home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go:107 +0x116
    	k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc00685ddc0?})
    		/home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go:64 +0x17b
    	panic({0x59381a0?, 0x91c6570?})
    		/usr/local/go/src/runtime/panic.go:783 +0x132
    	k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).bindClaim.func2()
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go:1247 +0x730
    	k8s.io/client-go/util/retry.OnError.func1()
    		/home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/retry/util.go:51 +0x30
    	k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection(0x9ebcca?)
    		/home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/wait/wait.go:150 +0x3e
    	k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff({0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x2, 0x0}, 0xc020507410)
    		/home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/wait/backoff.go:477 +0x5a
    	k8s.io/client-go/util/retry.OnError({0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc02c2d5380?, 0x4?)
    		/home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/retry/util.go:50 +0x96
    	k8s.io/client-go/util/retry.RetryOnConflict(...)
    		/home/prow/go/src/k8s.io/kubernetes/staging/src/k8s.io/client-go/util/retry/util.go:104
    	k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).bindClaim(0xc0024adb20, {0x69cea28, 0xc0061acbe0}, 0xc011876800, 0x0, 0xc025163408, {0xc021909d80, 0x8})
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go:1207 +0x845
    	k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).PreBind-range1(...)
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go:1073
    	k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).PreBind.(*claimStore).all.func2(...)
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/claims.go:72
    	k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources.(*DynamicResources).PreBind(0xc0024adb20, {0x69cea28, 0xc0061acbe0}, {0x69fc840?, 0xc0286f2540?}, 0xc025163408, {0xc021909d80, 0x8})
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go:1071 +0x246
    	k8s.io/kubernetes/pkg/scheduler/framework/runtime.(*frameworkImpl).runPreBindPlugin(0xc009628dc8, {0x69cea28, 0xc0061acbe0}, {0x7eabfddddb30, 0xc0024adb20}, {0x69fc840, 0xc0286f2540}, 0xc025163408, {0xc021909d80, 0x8})
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/runtime/framework.go:1532 +0x2e2
    	k8s.io/kubernetes/pkg/scheduler/framework/runtime.(*frameworkImpl).RunPreBindPlugins.func2({0x7eabfddddb30, 0xc0024adb20})
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/runtime/framework.go:1461 +0x1cf
    	k8s.io/kubernetes/pkg/scheduler/framework/runtime.(*frameworkImpl).RunPreBindPlugins(0xc009628dc8, {0x69cea28, 0xc0061ac690}, {0x69fc840, 0xc0286f2540}, 0xc025163408, {0xc021909d80, 0x8})
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/framework/runtime/framework.go:1484 +0x623
    	k8s.io/kubernetes/pkg/scheduler.(*Scheduler).bindingCycle(0xc02a6a7500, {0x69cea28, 0xc00f22a690}, {0x69fc840, 0xc0286f2540}, {0x6a32e60, 0xc009628dc8}, {{0xc021909d80, 0x8}, 0x8, ...}, ...)
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/schedule_one.go:457 +0x72a
    	k8s.io/kubernetes/pkg/scheduler.(*Scheduler).runBindingCycle(0xc02a6a7500, {0x69ce9f0?, 0xc0027e5470?}, {0x69fc840, 0xc0286f2540}, {0x6a32e60, 0xc009628dc8}, {{0xc021909d80, 0x8}, 0x8, ...}, ...)
    		/home/prow/go/src/k8s.io/kubernetes/pkg/scheduler/schedule_one.go:164 +0x1e8
This commit is contained in:
Patrick Ohly
2026-03-03 16:21:38 +01:00
parent 6be0185960
commit dd6f4d3a16
1 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/scheduler/framework/plugins/dynamicresources/dynamicresources.go
View File

@@ -1244,8 +1244,17 @@ func (pl *DynamicResources) bindClaim(ctx context.Context, state *stateData, ind
// preconditions. The apiserver will tell us with a
// non-conflict error if this isn't possible.
claim.Status.ReservedFor = append(claim.Status.ReservedFor, resourceapi.ResourceClaimConsumerReference{Resource: "pods", Name: pod.Name, UID: pod.UID})
if pl.fts.EnableDRADeviceBindingConditions && pl.fts.EnableDRAResourceClaimDeviceStatus && claim.Status.Allocation.AllocationTimestamp == nil {
claim.Status.Allocation.AllocationTimestamp = &metav1.Time{Time: time.Now()}
if pl.fts.EnableDRADeviceBindingConditions &&
pl.fts.EnableDRAResourceClaimDeviceStatus {
// Once the feature gate checks get removed, this code can be moved into an else branch
// for `if allocation != nil` above. For now it is kept here to avoid changing normal
// behavior.
if claim.Status.Allocation == nil {
return fmt.Errorf("claim %s got deallocated elsewhere in the meantime", klog.KObj(claim))
}
if claim.Status.Allocation.AllocationTimestamp == nil {
claim.Status.Allocation.AllocationTimestamp = &metav1.Time{Time: time.Now()}
}
}
updatedClaim, err := pl.clientset.ResourceV1().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
if err != nil {