From 368e699ed2e8d23d14a087221d94ceb46e86b1b3 Mon Sep 17 00:00:00 2001 From: Tsubasa Watanabe Date: Wed, 5 Nov 2025 21:41:59 +0900 Subject: [PATCH 1/2] scheduler: KEP-5007 move BindingConditions integration tests to separate file To improve maintainability and reduce the size of dra_test.go, this commit extracts the function "testDeviceBindingConditions" into a new file: binding_conditions_test.go. No functional changes are introduced. Signed-off-by: Tsubasa Watanabe --- .../dra/binding_conditions_test.go | 232 ++++++++++++++++++ test/integration/dra/dra_test.go | 204 --------------- 2 files changed, 232 insertions(+), 204 deletions(-) create mode 100644 test/integration/dra/binding_conditions_test.go diff --git a/test/integration/dra/binding_conditions_test.go b/test/integration/dra/binding_conditions_test.go new file mode 100644 index 00000000000..89cf8db1bcb --- /dev/null +++ b/test/integration/dra/binding_conditions_test.go @@ -0,0 +1,232 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "time" + + "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + resourceapi "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "k8s.io/kubernetes/test/utils/format" + "k8s.io/kubernetes/test/utils/ktesting" +) + +// testDeviceBindingConditions tests scheduling with mixed devices: one with BindingConditions, one without. +// It verifies that the scheduler prioritizes the device without BindingConditions for the first pod. +// The second pod then uses the device with BindingConditions. The test checks that the scheduler retries +// after an initial binding failure of the second pod, ensuring successful scheduling after rescheduling. +func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) { + namespace := createTestNamespace(tCtx, nil) + class, driverName := createTestClass(tCtx, namespace) + + nodeName := "worker-0" + poolWithBinding := nodeName + "-with-binding" + poolWithoutBinding := nodeName + "-without-binding" + bindingCondition := "attached" + failureCondition := "failed" + startScheduler(tCtx) + + slice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: namespace + "-", + }, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: &nodeName, + Pool: resourceapi.ResourcePool{ + Name: poolWithBinding, + ResourceSliceCount: 1, + }, + Driver: driverName, + Devices: []resourceapi.Device{ + { + Name: "with-binding", + BindingConditions: []string{bindingCondition}, + BindingFailureConditions: []string{failureCondition}, + }, + }, + }, + } + slice, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{FieldValidation: "Strict"}) + tCtx.ExpectNoError(err, "create slice") + + haveBindingConditionFields := len(slice.Spec.Devices[0].BindingConditions) > 0 || len(slice.Spec.Devices[0].BindingFailureConditions) > 0 + if !enabled { + if haveBindingConditionFields { + tCtx.Fatalf("Expected device binding condition fields to get dropped, got instead:\n%s", format.Object(slice, 1)) + } + return + } + if !haveBindingConditionFields { + tCtx.Fatalf("Expected device binding condition fields to be stored, got instead:\n%s", format.Object(slice, 1)) + } + + sliceWithoutBinding := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: namespace + "-without-binding-", + }, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: &nodeName, + Pool: resourceapi.ResourcePool{ + Name: poolWithoutBinding, + ResourceSliceCount: 1, + }, + Driver: driverName, + Devices: []resourceapi.Device{ + { + Name: "without-binding", + }, + }, + }, + } + _, err = tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, sliceWithoutBinding, metav1.CreateOptions{FieldValidation: "Strict"}) + tCtx.ExpectNoError(err, "create slice without binding conditions") + + // Schedule first pod and wait for the scheduler to reach the binding phase, which marks the claim as allocated. + start := time.Now() + claim1 := createClaim(tCtx, namespace, "-a", class, claim) + pod := createPod(tCtx, namespace, "-a", claim1, podWithClaimName) + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { + c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim1.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err) + claim1 = c + return claim1 + }).WithTimeout(10*time.Second).WithPolling(time.Second).Should(gomega.HaveField("Status.Allocation", gomega.Not(gomega.BeNil())), "Claim should have been allocated.") + end := time.Now() + gomega.NewWithT(tCtx).Expect(claim1).To(gomega.HaveField("Status.Allocation", gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{{ + Request: claim1.Spec.Devices.Requests[0].Name, + Driver: driverName, + Pool: poolWithoutBinding, + Device: "without-binding", + }}}), + // NodeSelector intentionally not checked - that's covered elsewhere. + "AllocationTimestamp": gomega.HaveField("Time", gomega.And( + gomega.BeTemporally(">=", start.Truncate(time.Second) /* may get rounded down during round-tripping */), + gomega.BeTemporally("<=", end), + )), + }))), "first allocated claim") + + waitForPodScheduled(tCtx, namespace, pod.Name) + + // Second pod should get the device with binding conditions. + claim2 := createClaim(tCtx, namespace, "-b", class, claim) + pod = createPod(tCtx, namespace, "-b", claim2, podWithClaimName) + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { + c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err) + claim2 = c + return claim2 + }).WithTimeout(10*time.Second).WithPolling(time.Second).Should(gomega.HaveField("Status.Allocation", gomega.Not(gomega.BeNil())), "Claim should have been allocated.") + end = time.Now() + gomega.NewWithT(tCtx).Expect(claim2).To(gomega.HaveField("Status.Allocation", gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{{ + Request: claim2.Spec.Devices.Requests[0].Name, + Driver: driverName, + Pool: poolWithBinding, + Device: "with-binding", + BindingConditions: []string{bindingCondition}, + BindingFailureConditions: []string{failureCondition}, + }}}), + // NodeSelector intentionally not checked - that's covered elsewhere. + "AllocationTimestamp": gomega.HaveField("Time", gomega.And( + gomega.BeTemporally(">=", start.Truncate(time.Second) /* may get rounded down during round-tripping */), + gomega.BeTemporally("<=", end), + )), + }))), "second allocated claim") + + // fail the binding condition for the second claim, so that it gets scheduled later. + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) + if err != nil { + return err + } + latest.Status.Devices = []resourceapi.AllocatedDeviceStatus{{ + Driver: driverName, + Pool: poolWithBinding, + Device: "with-binding", + Conditions: []metav1.Condition{{ + Type: failureCondition, + Status: metav1.ConditionTrue, + ObservedGeneration: latest.Generation, + LastTransitionTime: metav1.Now(), + Reason: "Testing", + Message: "The test has seen the allocation and is failing the binding.", + }}, + }} + _, err = tCtx.Client().ResourceV1().ResourceClaims(namespace).UpdateStatus(tCtx, latest, metav1.UpdateOptions{}) + return err + }) + tCtx.ExpectNoError(err, "add binding failure condition to second claim") + + // Wait until the claim.status.Devices[0].Conditions become nil again after rescheduling. + setConditionsFlag := false + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { + c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get claim") + claim2 = c + // Phase 1: saw conditions once + if claim2.Status.Devices != nil && len(claim2.Status.Devices[0].Conditions) != 0 { + setConditionsFlag = true + } + // Phase 2: after seeing conditions, wait until they are cleared AND allocation is present for the same device. + if setConditionsFlag { + // conditions cleared? + cleared := len(claim2.Status.Devices) == 0 + // allocation restored and matches the intended device? + allocated := claim2.Status.Allocation != nil && + len(claim2.Status.Allocation.Devices.Results) == 1 && + claim2.Status.Allocation.Devices.Results[0].Driver == driverName && + claim2.Status.Allocation.Devices.Results[0].Pool == poolWithBinding && + claim2.Status.Allocation.Devices.Results[0].Device == "with-binding" + if cleared && allocated { + return nil // done waiting + } + } + return claim2 + }).WithTimeout(30*time.Second).WithPolling(time.Second).Should(gomega.BeNil(), "claim should be re-allocated to with-binding before proceeding") + // Now it's safe to set the final binding condition. + // Allow the scheduler to proceed. + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) + if err != nil { + return err + } + // Write status.devices for the CURRENT allocation device. + latest.Status.Devices = []resourceapi.AllocatedDeviceStatus{{ + Driver: driverName, + Pool: poolWithBinding, + Device: "with-binding", + Conditions: []metav1.Condition{{ + Type: bindingCondition, + Status: metav1.ConditionTrue, + ObservedGeneration: latest.Generation, + LastTransitionTime: metav1.Now(), + Reason: "Testing", + Message: "The test has seen the allocation.", + }}, + }} + _, err = tCtx.Client().ResourceV1().ResourceClaims(namespace).UpdateStatus(tCtx, latest, metav1.UpdateOptions{}) + return err + }) + tCtx.ExpectNoError(err, "add binding condition to second claim") + waitForPodScheduled(tCtx, namespace, pod.Name) +} diff --git a/test/integration/dra/dra_test.go b/test/integration/dra/dra_test.go index 866794c5a02..3209c129c07 100644 --- a/test/integration/dra/dra_test.go +++ b/test/integration/dra/dra_test.go @@ -53,7 +53,6 @@ import ( resourceapiac "k8s.io/client-go/applyconfigurations/resource/v1" "k8s.io/client-go/informers" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/retry" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/metrics/testutil" @@ -1599,209 +1598,6 @@ func matchPointer[T any](p *T) gtypes.GomegaMatcher { return gstruct.PointTo(gomega.Equal(*p)) } -// testDeviceBindingConditions tests scheduling with mixed devices: one with BindingConditions, one without. -// It verifies that the scheduler prioritizes the device without BindingConditions for the first pod. -// The second pod then uses the device with BindingConditions. The test checks that the scheduler retries -// after an initial binding failure of the second pod, ensuring successful scheduling after rescheduling. -func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) { - namespace := createTestNamespace(tCtx, nil) - class, driverName := createTestClass(tCtx, namespace) - - nodeName := "worker-0" - poolWithBinding := nodeName + "-with-binding" - poolWithoutBinding := nodeName + "-without-binding" - bindingCondition := "attached" - failureCondition := "failed" - startScheduler(tCtx) - - slice := &resourceapi.ResourceSlice{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: namespace + "-", - }, - Spec: resourceapi.ResourceSliceSpec{ - NodeName: &nodeName, - Pool: resourceapi.ResourcePool{ - Name: poolWithBinding, - ResourceSliceCount: 1, - }, - Driver: driverName, - Devices: []resourceapi.Device{ - { - Name: "with-binding", - BindingConditions: []string{bindingCondition}, - BindingFailureConditions: []string{failureCondition}, - }, - }, - }, - } - slice, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{FieldValidation: "Strict"}) - tCtx.ExpectNoError(err, "create slice") - - haveBindingConditionFields := len(slice.Spec.Devices[0].BindingConditions) > 0 || len(slice.Spec.Devices[0].BindingFailureConditions) > 0 - if !enabled { - if haveBindingConditionFields { - tCtx.Fatalf("Expected device binding condition fields to get dropped, got instead:\n%s", format.Object(slice, 1)) - } - return - } - if !haveBindingConditionFields { - tCtx.Fatalf("Expected device binding condition fields to be stored, got instead:\n%s", format.Object(slice, 1)) - } - - sliceWithoutBinding := &resourceapi.ResourceSlice{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: namespace + "-without-binding-", - }, - Spec: resourceapi.ResourceSliceSpec{ - NodeName: &nodeName, - Pool: resourceapi.ResourcePool{ - Name: poolWithoutBinding, - ResourceSliceCount: 1, - }, - Driver: driverName, - Devices: []resourceapi.Device{ - { - Name: "without-binding", - }, - }, - }, - } - _, err = tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, sliceWithoutBinding, metav1.CreateOptions{FieldValidation: "Strict"}) - tCtx.ExpectNoError(err, "create slice without binding conditions") - - // Schedule first pod and wait for the scheduler to reach the binding phase, which marks the claim as allocated. - start := time.Now() - claim1 := createClaim(tCtx, namespace, "-a", class, claim) - pod := createPod(tCtx, namespace, "-a", podWithClaimName, claim1) - ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { - c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim1.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err) - claim1 = c - return claim1 - }).WithTimeout(10*time.Second).WithPolling(time.Second).Should(gomega.HaveField("Status.Allocation", gomega.Not(gomega.BeNil())), "Claim should have been allocated.") - end := time.Now() - gomega.NewWithT(tCtx).Expect(claim1).To(gomega.HaveField("Status.Allocation", gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ - Results: []resourceapi.DeviceRequestAllocationResult{{ - Request: claim1.Spec.Devices.Requests[0].Name, - Driver: driverName, - Pool: poolWithoutBinding, - Device: "without-binding", - }}}), - // NodeSelector intentionally not checked - that's covered elsewhere. - "AllocationTimestamp": gomega.HaveField("Time", gomega.And( - gomega.BeTemporally(">=", start.Truncate(time.Second) /* may get rounded down during round-tripping */), - gomega.BeTemporally("<=", end), - )), - }))), "first allocated claim") - - waitForPodScheduled(tCtx, namespace, pod.Name) - - // Second pod should get the device with binding conditions. - claim2 := createClaim(tCtx, namespace, "-b", class, claim) - pod = createPod(tCtx, namespace, "-b", podWithClaimName, claim2) - ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { - c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err) - claim2 = c - return claim2 - }).WithTimeout(10*time.Second).WithPolling(time.Second).Should(gomega.HaveField("Status.Allocation", gomega.Not(gomega.BeNil())), "Claim should have been allocated.") - end = time.Now() - gomega.NewWithT(tCtx).Expect(claim2).To(gomega.HaveField("Status.Allocation", gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ - Results: []resourceapi.DeviceRequestAllocationResult{{ - Request: claim2.Spec.Devices.Requests[0].Name, - Driver: driverName, - Pool: poolWithBinding, - Device: "with-binding", - BindingConditions: []string{bindingCondition}, - BindingFailureConditions: []string{failureCondition}, - }}}), - // NodeSelector intentionally not checked - that's covered elsewhere. - "AllocationTimestamp": gomega.HaveField("Time", gomega.And( - gomega.BeTemporally(">=", start.Truncate(time.Second) /* may get rounded down during round-tripping */), - gomega.BeTemporally("<=", end), - )), - }))), "second allocated claim") - - // fail the binding condition for the second claim, so that it gets scheduled later. - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - latest, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) - if err != nil { - return err - } - latest.Status.Devices = []resourceapi.AllocatedDeviceStatus{{ - Driver: driverName, - Pool: poolWithBinding, - Device: "with-binding", - Conditions: []metav1.Condition{{ - Type: failureCondition, - Status: metav1.ConditionTrue, - ObservedGeneration: latest.Generation, - LastTransitionTime: metav1.Now(), - Reason: "Testing", - Message: "The test has seen the allocation and is failing the binding.", - }}, - }} - _, err = tCtx.Client().ResourceV1().ResourceClaims(namespace).UpdateStatus(tCtx, latest, metav1.UpdateOptions{}) - return err - }) - tCtx.ExpectNoError(err, "add binding failure condition to second claim") - - // Wait until the claim.status.Devices[0].Conditions become nil again after rescheduling. - setConditionsFlag := false - ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { - c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get claim") - claim2 = c - // Phase 1: saw conditions once - if claim2.Status.Devices != nil && len(claim2.Status.Devices[0].Conditions) != 0 { - setConditionsFlag = true - } - // Phase 2: after seeing conditions, wait until they are cleared AND allocation is present for the same device. - if setConditionsFlag { - // conditions cleared? - cleared := len(claim2.Status.Devices) == 0 - // allocation restored and matches the intended device? - allocated := claim2.Status.Allocation != nil && - len(claim2.Status.Allocation.Devices.Results) == 1 && - claim2.Status.Allocation.Devices.Results[0].Driver == driverName && - claim2.Status.Allocation.Devices.Results[0].Pool == poolWithBinding && - claim2.Status.Allocation.Devices.Results[0].Device == "with-binding" - if cleared && allocated { - return nil // done waiting - } - } - return claim2 - }).WithTimeout(30*time.Second).WithPolling(time.Second).Should(gomega.BeNil(), "claim should be re-allocated to with-binding before proceeding") - // Now it's safe to set the final binding condition. - // Allow the scheduler to proceed. - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - latest, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) - if err != nil { - return err - } - // Write status.devices for the CURRENT allocation device. - latest.Status.Devices = []resourceapi.AllocatedDeviceStatus{{ - Driver: driverName, - Pool: poolWithBinding, - Device: "with-binding", - Conditions: []metav1.Condition{{ - Type: bindingCondition, - Status: metav1.ConditionTrue, - ObservedGeneration: latest.Generation, - LastTransitionTime: metav1.Now(), - Reason: "Testing", - Message: "The test has seen the allocation.", - }}, - }} - _, err = tCtx.Client().ResourceV1().ResourceClaims(namespace).UpdateStatus(tCtx, latest, metav1.UpdateOptions{}) - return err - }) - tCtx.ExpectNoError(err, "add binding condition to second claim") - waitForPodScheduled(tCtx, namespace, pod.Name) -} - func testInvalidResourceSlices(tCtx ktesting.TContext) { driverNamePlaceholder := "driver" startScheduler(tCtx) From 1225ce509e6f14d09050e91efe4d9c2443e8f66d Mon Sep 17 00:00:00 2001 From: Tsubasa Watanabe Date: Wed, 5 Nov 2025 21:55:32 +0900 Subject: [PATCH 2/2] scheduler: KEP-5007 add integration tests - Rescheduling on binding failure using taints or device removal - Triggering binding timeout when BindingConditions remain unmet - Recovery to a device without BindingConditions after binding timeout Signed-off-by: Tsubasa Watanabe --- .../dra/binding_conditions_test.go | 472 ++++++++++++++++-- test/integration/dra/helpers_test.go | 25 + 2 files changed, 449 insertions(+), 48 deletions(-) diff --git a/test/integration/dra/binding_conditions_test.go b/test/integration/dra/binding_conditions_test.go index 89cf8db1bcb..83f4acfde7e 100644 --- a/test/integration/dra/binding_conditions_test.go +++ b/test/integration/dra/binding_conditions_test.go @@ -17,10 +17,13 @@ limitations under the License. package dra import ( + "fmt" + "strings" "time" "github.com/onsi/gomega" "github.com/onsi/gomega/gstruct" + v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -28,19 +31,33 @@ import ( "k8s.io/kubernetes/test/utils/ktesting" ) -// testDeviceBindingConditions tests scheduling with mixed devices: one with BindingConditions, one without. +var ( + nodeName = "worker-0" + poolWithBinding = nodeName + "-with-binding" + poolWithoutBinding = nodeName + "-without-binding" + bindingCondition = "attached" + failureCondition = "failed" +) + +// testDeviceBindingConditions is the entry point for running each integration test that verifies DeviceBindingConditions. +// Some of these tests use device taints, and they assume that DRADeviceTaints is enabled. +func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) { + tCtx.Run("BasicFlow", func(tCtx ktesting.TContext) { testDeviceBindingConditionsBasicFlow(tCtx, enabled) }) + if enabled { + tCtx.Run("FailureTaints", func(tCtx ktesting.TContext) { testDeviceBindingFailureConditionsReschedule(tCtx, true) }) + tCtx.Run("FailureRemove", func(tCtx ktesting.TContext) { testDeviceBindingFailureConditionsReschedule(tCtx, false) }) + tCtx.Run("TimeoutReached", func(tCtx ktesting.TContext) { testDeviceBindingConditionsTimeoutReached(tCtx) }) + tCtx.Run("TimeoutRecover", func(tCtx ktesting.TContext) { testDeviceBindingConditionsTimeoutRecovery(tCtx) }) + } +} + +// testBindingConditionsBasicFlow tests scheduling with mixed devices: one with BindingConditions, one without. // It verifies that the scheduler prioritizes the device without BindingConditions for the first pod. // The second pod then uses the device with BindingConditions. The test checks that the scheduler retries // after an initial binding failure of the second pod, ensuring successful scheduling after rescheduling. -func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) { +func testDeviceBindingConditionsBasicFlow(tCtx ktesting.TContext, enabled bool) { namespace := createTestNamespace(tCtx, nil) class, driverName := createTestClass(tCtx, namespace) - - nodeName := "worker-0" - poolWithBinding := nodeName + "-with-binding" - poolWithoutBinding := nodeName + "-without-binding" - bindingCondition := "attached" - failureCondition := "failed" startScheduler(tCtx) slice := &resourceapi.ResourceSlice{ @@ -101,13 +118,8 @@ func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) { // Schedule first pod and wait for the scheduler to reach the binding phase, which marks the claim as allocated. start := time.Now() claim1 := createClaim(tCtx, namespace, "-a", class, claim) - pod := createPod(tCtx, namespace, "-a", claim1, podWithClaimName) - ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { - c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim1.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err) - claim1 = c - return claim1 - }).WithTimeout(10*time.Second).WithPolling(time.Second).Should(gomega.HaveField("Status.Allocation", gomega.Not(gomega.BeNil())), "Claim should have been allocated.") + pod := createPod(tCtx, namespace, "-a", podWithClaimName, claim1) + claim1 = waitForClaimAllocatedToDevice(tCtx, namespace, claim1.Name, 10*time.Second) end := time.Now() gomega.NewWithT(tCtx).Expect(claim1).To(gomega.HaveField("Status.Allocation", gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ @@ -128,13 +140,8 @@ func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) { // Second pod should get the device with binding conditions. claim2 := createClaim(tCtx, namespace, "-b", class, claim) - pod = createPod(tCtx, namespace, "-b", claim2, podWithClaimName) - ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { - c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err) - claim2 = c - return claim2 - }).WithTimeout(10*time.Second).WithPolling(time.Second).Should(gomega.HaveField("Status.Allocation", gomega.Not(gomega.BeNil())), "Claim should have been allocated.") + pod = createPod(tCtx, namespace, "-b", podWithClaimName, claim2) + claim2 = waitForClaimAllocatedToDevice(tCtx, namespace, claim2.Name, 10*time.Second) end = time.Now() gomega.NewWithT(tCtx).Expect(claim2).To(gomega.HaveField("Status.Allocation", gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ @@ -177,32 +184,15 @@ func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) { }) tCtx.ExpectNoError(err, "add binding failure condition to second claim") - // Wait until the claim.status.Devices[0].Conditions become nil again after rescheduling. - setConditionsFlag := false - ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { - c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim2.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get claim") - claim2 = c - // Phase 1: saw conditions once - if claim2.Status.Devices != nil && len(claim2.Status.Devices[0].Conditions) != 0 { - setConditionsFlag = true - } - // Phase 2: after seeing conditions, wait until they are cleared AND allocation is present for the same device. - if setConditionsFlag { - // conditions cleared? - cleared := len(claim2.Status.Devices) == 0 - // allocation restored and matches the intended device? - allocated := claim2.Status.Allocation != nil && - len(claim2.Status.Allocation.Devices.Results) == 1 && - claim2.Status.Allocation.Devices.Results[0].Driver == driverName && - claim2.Status.Allocation.Devices.Results[0].Pool == poolWithBinding && - claim2.Status.Allocation.Devices.Results[0].Device == "with-binding" - if cleared && allocated { - return nil // done waiting - } - } - return claim2 - }).WithTimeout(30*time.Second).WithPolling(time.Second).Should(gomega.BeNil(), "claim should be re-allocated to with-binding before proceeding") + // Then wait until the scheduler has cleared the device statuses again. + waitForClaim(tCtx, namespace, claim2.Name, 30*time.Second, + gomega.HaveField("Status.Devices", gomega.HaveLen(0)), + "claim should have cleared device conditions after rescheduling", + ) + + // allocation restored? + claim2 = waitForClaimAllocatedToDevice(tCtx, namespace, claim2.Name, 10*time.Second) + // Now it's safe to set the final binding condition. // Allow the scheduler to proceed. err = retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -230,3 +220,389 @@ func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) { tCtx.ExpectNoError(err, "add binding condition to second claim") waitForPodScheduled(tCtx, namespace, pod.Name) } + +// testBindingFailureReschedule verifies scheduling behavior when device preparation fails on a node. +// It tests that a BindingFailure is written, and the scheduler successfully reschedules the pod +// to a different node where binding succeeds. This ensures that failure recovery via rescheduling works as expected. +// Device preparation failure is simulated in two ways: by applying DeviceTaints or by removing the device from ResourceSlice. +// The simulation method is controlled via the `useTaints` argument: when true, DeviceTaints are used; when false, the device is removed from ResourceSlice. +func testDeviceBindingFailureConditionsReschedule(tCtx ktesting.TContext, useTaints bool) { + namespace := createTestNamespace(tCtx, nil) + class, driverName := createTestClass(tCtx, namespace) + startScheduler(tCtx) + + anotherNodeName := "worker-1" + anotherPoolWithoutBinding := anotherNodeName + "-without-binding" + + slice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: namespace + "-", + }, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: &nodeName, + Pool: resourceapi.ResourcePool{ + Name: poolWithBinding, + ResourceSliceCount: 1, + }, + Driver: driverName, + Devices: []resourceapi.Device{ + { + Name: "with-binding", + BindingConditions: []string{bindingCondition}, + BindingFailureConditions: []string{failureCondition}, + }, + }, + }, + } + slice = createSlice(tCtx, slice) + + // Schedule the first pod to a device that has binding conditions set, + // ensuring the initial allocation occurs on the intended node. + claim1 := createClaim(tCtx, namespace, "-a", class, claim) + pod := createPod(tCtx, namespace, "-a", podWithClaimName, claim1) + claim1 = waitForClaimAllocatedToDevice(tCtx, namespace, claim1.Name, 10*time.Second) + gomega.NewWithT(tCtx).Expect(claim1).To(gomega.HaveField("Status.Allocation", gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{{ + Request: claim1.Spec.Devices.Requests[0].Name, + Driver: driverName, + Pool: poolWithBinding, + Device: "with-binding", + BindingConditions: []string{bindingCondition}, + BindingFailureConditions: []string{failureCondition}, + }}}), + }))), "third allocated claim to the device with binding conditions") + + if useTaints { + // Add taint to the device with binding conditions, + // preventing further scheduling to this device. + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest, err := tCtx.Client().ResourceV1().ResourceSlices().Get(tCtx, slice.Name, metav1.GetOptions{}) + if err != nil { + return err + } + slice = latest.DeepCopy() + slice.Spec.Devices[0].Taints = []resourceapi.DeviceTaint{ + { + Key: "dra-test.k8s.io/preparation-failed", + Value: "true", + Effect: resourceapi.DeviceTaintEffectNoSchedule, + }, + } + _, err = tCtx.Client().ResourceV1().ResourceSlices().Update(tCtx, slice, metav1.UpdateOptions{}) + return err + }) + tCtx.ExpectNoError(err, "add taint to second slice") + } else { + // Remove the device from the slice to simulate its unavailability due to preparation failure. + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest, err := tCtx.Client().ResourceV1().ResourceSlices().Get(tCtx, slice.Name, metav1.GetOptions{}) + if err != nil { + return err + } + slice = latest.DeepCopy() + slice.Spec.Devices = nil + _, err = tCtx.Client().ResourceV1().ResourceSlices().Update(tCtx, slice, metav1.UpdateOptions{}) + return err + }) + tCtx.ExpectNoError(err, "remove devices in slice") + } + + // Create a new slice on a different node with a device that has no binding conditions, + // allowing the scheduler to retry and allocate the claim successfully. + sliceWithoutBinding := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: namespace + "-without-binding-", + }, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: &anotherNodeName, + Pool: resourceapi.ResourcePool{ + Name: anotherPoolWithoutBinding, + ResourceSliceCount: 1, + }, + Driver: driverName, + Devices: []resourceapi.Device{ + { + Name: "without-binding", + }, + }, + }, + } + createSlice(tCtx, sliceWithoutBinding) + + // Explicitly fail the binding condition for the third claim to trigger rescheduling logic. + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim1.Name, metav1.GetOptions{}) + if err != nil { + return err + } + claim1 = latest.DeepCopy() + claim1.Status.Devices = []resourceapi.AllocatedDeviceStatus{{ + Driver: driverName, + Pool: poolWithBinding, + Device: "with-binding", + Conditions: []metav1.Condition{{ + Type: failureCondition, + Status: metav1.ConditionTrue, + ObservedGeneration: claim1.Generation, + LastTransitionTime: metav1.Now(), + Reason: "Testing", + Message: "The test has seen the allocation and is failing the binding.", + }}, + }} + _, err = tCtx.Client().ResourceV1().ResourceClaims(namespace).UpdateStatus(tCtx, claim1, metav1.UpdateOptions{}) + return err + }) + tCtx.ExpectNoError(err, "add binding failure condition to claim") + + // Then wait until the scheduler has cleared the device statuses again. + waitForClaim(tCtx, namespace, claim1.Name, 30*time.Second, + gomega.HaveField("Status.Devices", gomega.HaveLen(0)), + "claim should have cleared device conditions after rescheduling", + ) + + // allocation restored? + claim1 = waitForClaimAllocatedToDevice(tCtx, namespace, claim1.Name, 10*time.Second) + + gomega.NewWithT(tCtx).Expect(claim1).To(gomega.HaveField("Status.Allocation", gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{{ + Request: claim1.Spec.Devices.Requests[0].Name, + Driver: driverName, + Pool: anotherPoolWithoutBinding, + Device: "without-binding", + }}}), + }))), "third allocated claim to the device without binding conditions") + + waitForPodScheduled(tCtx, namespace, pod.Name) +} + +// testBindingConditionsTimeoutReachedd verifies that a short bindingTimeout triggers +// a PreBind timeout when the required BindingConditions never become true. +func testDeviceBindingConditionsTimeoutReached(tCtx ktesting.TContext) { + namespace := createTestNamespace(tCtx, nil) + class, driver := createTestClass(tCtx, namespace) + + // One device that REQUIRES a binding condition. + slice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{GenerateName: namespace + "-timeout-"}, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: &nodeName, + Pool: resourceapi.ResourcePool{ + Name: poolWithBinding, + ResourceSliceCount: 1, + }, + Driver: driver, + Devices: []resourceapi.Device{{ + Name: "with-binding", + BindingConditions: []string{bindingCondition}, + BindingFailureConditions: []string{failureCondition}, + }}, + }, + } + createSlice(tCtx, slice) + + wantTO := 6 * time.Second // bindingTimeout + maxTO := wantTO * 19 / 10 // 1.9 * wantTO + + // Start the scheduler with a short binding timeout. + cfg := fmt.Sprintf(` +profiles: +- schedulerName: default-scheduler + pluginConfig: + - name: DynamicResources + args: + bindingTimeout: %s +`, wantTO) + + startSchedulerWithConfig(tCtx, cfg) + + // Create claim+pod: allocation happens, then scheduler waits in PreBind. + claim1 := createClaim(tCtx, namespace, "-timeout-enforced", class, claim) + pod := createPod(tCtx, namespace, "-timeout-enforced", podWithClaimName, claim1) + + // Wait until the claim is allocated. + allocatedClaim := waitForClaimAllocatedToDevice(tCtx, namespace, claim1.Name, 10*time.Second) + + gomega.NewWithT(tCtx).Expect(allocatedClaim).To(gomega.HaveField( + "Status.Allocation", + gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{{ + Request: allocatedClaim.Spec.Devices.Requests[0].Name, + Driver: driver, + Pool: poolWithBinding, + Device: "with-binding", + BindingConditions: []string{bindingCondition}, + BindingFailureConditions: []string{failureCondition}, + }}, + }), + }), + )), "Claim must be allocated to the condition-gated device") + + tStart := time.Now() + + // The scheduler should hit the binding timeout and surface that on the pod. + // We poll the pod's conditions until we see a message containing "binding timeout". + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *v1.Pod { + p, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod") + return p + }).WithTimeout(maxTO).WithPolling(300*time.Millisecond).Should( + gomega.HaveField("Status.Conditions", + gomega.ContainElement( + gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Message": gomega.ContainSubstring("binding timeout"), + }), + ), + ), + "pod should report binding timeout in a condition message", + ) + + elapsed := time.Since(tStart) + gomega.NewWithT(tCtx).Expect(elapsed).To( + gomega.BeNumerically("<=", maxTO), + "bindingTimeout should trigger roughly near %s (observed %v)", wantTO, elapsed, + ) + // Verify that the pod remains unscheduled after the binding timeout. + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *v1.Pod { + pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod") + return pod + }).WithTimeout(wantTO).WithPolling(200 * time.Millisecond).Should(gomega.SatisfyAll( + gomega.HaveField("Spec.NodeName", gomega.BeEmpty()), + + gomega.HaveField("Status.Conditions", + gomega.Not(gomega.ContainElement(v1.PodCondition{ + Type: v1.PodScheduled, + Status: v1.ConditionTrue, + }))), + )) +} + +// testDeviceBindingConditionsTimeoutRecovery verifies that when a device with BindingConditions +// fails to become ready within the timeout (BindingTimeout enforced), and a new device without +// binding conditions is added, the scheduler reschedules the claim to the new available device. +func testDeviceBindingConditionsTimeoutRecovery(tCtx ktesting.TContext) { + namespace := createTestNamespace(tCtx, nil) + class, driverName := createTestClass(tCtx, namespace) + + // Start the scheduler with a short binding timeout. + const cfg = ` +profiles: +- schedulerName: default-scheduler + pluginConfig: + - name: DynamicResources + args: + bindingTimeout: 5s +` + startSchedulerWithConfig(tCtx, cfg) + + // Initial slice: one device that *requires* a binding condition that never becomes true. + slice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: namespace + "-", + }, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: &nodeName, + Pool: resourceapi.ResourcePool{ + Name: poolWithBinding, + ResourceSliceCount: 1, + }, + Driver: driverName, + Devices: []resourceapi.Device{{ + Name: "with-binding", + BindingConditions: []string{bindingCondition}, + BindingFailureConditions: []string{failureCondition}, + }}, + }, + } + createSlice(tCtx, slice) + claim1 := createClaim(tCtx, namespace, "-timeout", class, claim) + pod := createPod(tCtx, namespace, "-timeout", podWithClaimName, claim1) + + claim1 = waitForClaimAllocatedToDevice(tCtx, namespace, claim1.Name, 10*time.Second) + gomega.NewWithT(tCtx).Expect(claim1).To(gomega.HaveField( + "Status.Allocation", + gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{{ + Request: claim.Spec.Devices.Requests[0].Name, + Driver: driverName, + Pool: poolWithBinding, + Device: "with-binding", + BindingConditions: []string{bindingCondition}, + BindingFailureConditions: []string{failureCondition}, + }}, + }), + }), + )), "Expected allocation to the condition-gated device") + + sliceWithoutBinding := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{GenerateName: namespace + "-recovery-"}, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: &nodeName, + Pool: resourceapi.ResourcePool{ + Name: poolWithoutBinding, + ResourceSliceCount: 1, + }, + Driver: driverName, + Devices: []resourceapi.Device{{ + Name: "without-binding", + }}, + }, + } + sliceWithoutBinding = createSlice(tCtx, sliceWithoutBinding) + + // Ensure the ResourceSlice has been created before the binding timeout occurs. + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) error { + p, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) + if err == nil { + for _, c := range p.Status.Conditions { + if strings.Contains(strings.ToLower(c.Message), "binding timeout") { + return gomega.StopTrying("binding timeout occurred before slice is created") + } + } + } + _, err = tCtx.Client().ResourceV1().ResourceSlices().Get(tCtx, sliceWithoutBinding.Name, metav1.GetOptions{}) + return err + }).WithTimeout(10*time.Second).WithPolling(300*time.Millisecond).Should( + gomega.Succeed(), "slice must be created before binding timeout") + + // Wait until the binding timeout occurs. + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *v1.Pod { + p, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod") + return p + }).WithTimeout(20*time.Second).WithPolling(300*time.Millisecond).Should( + gomega.HaveField("Status.Conditions", + gomega.ContainElement( + gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Message": gomega.ContainSubstring("binding timeout"), + }), + ), + ), + "pod should report binding timeout before reallocation", + ) + + // Verify recovery to the newly added device without BindingConditions through rescheduling triggered by binding timeout. + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { + c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claim1.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err) + return c + }).WithTimeout(10*time.Second).WithPolling(1*time.Second).Should(gomega.HaveField( + "Status.Allocation", + gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Devices": gomega.Equal(resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{{ + Request: claim.Spec.Devices.Requests[0].Name, + Driver: driverName, + Pool: poolWithoutBinding, + Device: "without-binding", + }}, + }), + }), + )), "Expected allocation to the device without binding conditions") + + waitForPodScheduled(tCtx, namespace, pod.Name) +} diff --git a/test/integration/dra/helpers_test.go b/test/integration/dra/helpers_test.go index d903ce06ddc..32484053572 100644 --- a/test/integration/dra/helpers_test.go +++ b/test/integration/dra/helpers_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/onsi/gomega" + gtypes "github.com/onsi/gomega/types" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" @@ -202,3 +203,27 @@ func waitForNotFound[T any](tCtx ktesting.TContext, get func(context.Context, st return err }).WithTimeout(60*time.Second).Should(gomega.MatchError(apierrors.IsNotFound, "IsNotFound"), "Object %T %s should have been removed.", t, name) } + +func waitForClaim(tCtx ktesting.TContext, namespace, claimName string, timeout time.Duration, match gtypes.GomegaMatcher, description ...any) *resourceapi.ResourceClaim { + tCtx.Helper() + var latestClaim *resourceapi.ResourceClaim + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *resourceapi.ResourceClaim { + c, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claimName, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get claim") + latestClaim = c + return latestClaim + }).WithTimeout(timeout).WithPolling(time.Second).Should(match, description...) + return latestClaim +} + +func waitForClaimAllocatedToDevice(tCtx ktesting.TContext, namespace, claimName string, timeout time.Duration) *resourceapi.ResourceClaim { + tCtx.Helper() + return waitForClaim( + tCtx, + namespace, + claimName, + timeout, + gomega.HaveField("Status.Allocation", gomega.Not(gomega.BeNil())), + "Claim should have been allocated.", + ) +}