From aa3f79d4c96f7d2da6b8b7ca977d4e6d5f7caa42 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 12 Dec 2025 08:44:48 +0100 Subject: [PATCH 1/2] DRA upgrade/downgrade: add DeviceTaints test This automatically tests a few scenarios across cluster upgrade/downgrade. --- test/e2e/dra/utils/builder.go | 34 ++++--- test/e2e/dra/utils/deploy.go | 1 + test/e2e_dra/devicetaints_test.go | 137 ++++++++++++++++++++++++++ test/e2e_dra/upgradedowngrade_test.go | 11 ++- 4 files changed, 164 insertions(+), 19 deletions(-) create mode 100644 test/e2e_dra/devicetaints_test.go diff --git a/test/e2e/dra/utils/builder.go b/test/e2e/dra/utils/builder.go index 47adf5f0d7f..82c6cabff91 100644 --- a/test/e2e/dra/utils/builder.go +++ b/test/e2e/dra/utils/builder.go @@ -60,7 +60,7 @@ func (b *Builder) ExtendedResourceName(i int) string { case SingletonIndex: return e2enode.SampleDeviceResourceName default: - return b.driver.Name + "/resource" + fmt.Sprintf("-%d", i) + return b.Driver.Name + "/resource" + fmt.Sprintf("-%d", i) } } @@ -68,7 +68,7 @@ func (b *Builder) ExtendedResourceName(i int) string { // namespace. type Builder struct { namespace string - driver *Driver + Driver *Driver UseExtendedResourceName bool podCounter int @@ -79,7 +79,7 @@ type Builder struct { // ClassName returns the default device class name. func (b *Builder) ClassName() string { - return b.namespace + b.driver.NameSuffix + "-class" + return b.namespace + b.Driver.NameSuffix + "-class" } // SingletonIndex causes Builder.Class and ExtendedResourceName to create a @@ -115,14 +115,14 @@ func (b *Builder) Class(i int) *resourceapi.DeviceClass { } class.Spec.Selectors = []resourceapi.DeviceSelector{{ CEL: &resourceapi.CELDeviceSelector{ - Expression: fmt.Sprintf(`device.driver == "%s"`, b.driver.Name), + Expression: fmt.Sprintf(`device.driver == "%s"`, b.Driver.Name), }, }} if b.ClassParameters != "" { class.Spec.Config = []resourceapi.DeviceClassConfiguration{{ DeviceConfiguration: resourceapi.DeviceConfiguration{ Opaque: &resourceapi.OpaqueDeviceConfiguration{ - Driver: b.driver.Name, + Driver: b.Driver.Name, Parameters: runtime.RawExtension{Raw: []byte(b.ClassParameters)}, }, }, @@ -135,7 +135,7 @@ func (b *Builder) Class(i int) *resourceapi.DeviceClass { // that test pods can reference func (b *Builder) ExternalClaim() *resourceapi.ResourceClaim { b.claimCounter++ - name := "external-claim" + b.driver.NameSuffix // This is what podExternal expects. + name := "external-claim" + b.Driver.NameSuffix // This is what podExternal expects. if b.claimCounter > 1 { name += fmt.Sprintf("-%d", b.claimCounter) } @@ -160,7 +160,7 @@ func (b *Builder) claimSpecWithV1beta1() resourcev1beta1.ResourceClaimSpec { Config: []resourcev1beta1.DeviceClaimConfiguration{{ DeviceConfiguration: resourcev1beta1.DeviceConfiguration{ Opaque: &resourcev1beta1.OpaqueDeviceConfiguration{ - Driver: b.driver.Name, + Driver: b.Driver.Name, Parameters: runtime.RawExtension{ Raw: []byte(parameters), }, @@ -188,7 +188,7 @@ func (b *Builder) claimSpecWithV1beta2() resourcev1beta2.ResourceClaimSpec { Config: []resourcev1beta2.DeviceClaimConfiguration{{ DeviceConfiguration: resourcev1beta2.DeviceConfiguration{ Opaque: &resourcev1beta2.OpaqueDeviceConfiguration{ - Driver: b.driver.Name, + Driver: b.Driver.Name, Parameters: runtime.RawExtension{ Raw: []byte(parameters), }, @@ -216,7 +216,7 @@ func (b *Builder) ClaimSpec() resourceapi.ResourceClaimSpec { Config: []resourceapi.DeviceClaimConfiguration{{ DeviceConfiguration: resourceapi.DeviceConfiguration{ Opaque: &resourceapi.OpaqueDeviceConfiguration{ - Driver: b.driver.Name, + Driver: b.Driver.Name, Parameters: runtime.RawExtension{ Raw: []byte(parameters), }, @@ -249,7 +249,7 @@ func (b *Builder) Pod() *v1.Pod { pod.Spec.RestartPolicy = v1.RestartPolicyNever pod.GenerateName = "" b.podCounter++ - pod.Name = fmt.Sprintf("tester%s-%d", b.driver.NameSuffix, b.podCounter) + pod.Name = fmt.Sprintf("tester%s-%d", b.Driver.NameSuffix, b.podCounter) return pod } @@ -314,11 +314,15 @@ func (b *Builder) PodInlineMultiple() (*v1.Pod, *resourceapi.ResourceClaimTempla } // PodExternal adds a pod that references external resource claim with default class name and parameters. +// +// Note that this references *the initial* result of ExternalClaim. When generating multiple such +// external claims, pod.Spec.ResourceClaims[0].ResourceClaimName must be adapted by the caller, +// if desired. func (b *Builder) PodExternal() *v1.Pod { pod := b.Pod() pod.Spec.Containers[0].Name = "with-resource" podClaimName := "resource-claim" - externalClaimName := "external-claim" + b.driver.NameSuffix + externalClaimName := "external-claim" + b.Driver.NameSuffix pod.Spec.ResourceClaims = []v1.PodResourceClaim{ { Name: podClaimName, @@ -417,7 +421,7 @@ func (b *Builder) DeletePodAndWaitForNotFound(tCtx ktesting.TContext, pod *v1.Po func (b *Builder) TestPod(tCtx ktesting.TContext, pod *v1.Pod, env ...string) { tCtx.Helper() - if !b.driver.WithKubelet { + if !b.Driver.WithKubelet { // Less testing when we cannot rely on the kubelet to actually run the pod. err := e2epod.WaitForPodScheduled(tCtx, tCtx.Client(), pod.Namespace, pod.Name) tCtx.ExpectNoError(err, "schedule pod") @@ -474,7 +478,7 @@ func TestContainerEnv(tCtx ktesting.TContext, pod *v1.Pod, containerName string, } func NewBuilder(f *framework.Framework, driver *Driver) *Builder { - b := &Builder{driver: driver} + b := &Builder{Driver: driver} ginkgo.BeforeEach(func() { b.setUp(f.TContext(context.Background())) }) @@ -482,7 +486,7 @@ func NewBuilder(f *framework.Framework, driver *Driver) *Builder { } func NewBuilderNow(tCtx ktesting.TContext, driver *Driver) *Builder { - b := &Builder{driver: driver} + b := &Builder{Driver: driver} b.setUp(tCtx) return b } @@ -542,7 +546,7 @@ func (b *Builder) tearDown(tCtx ktesting.TContext) { } } - for host, plugin := range b.driver.Nodes { + for host, plugin := range b.Driver.Nodes { tCtx.Logf("Waiting for resources on %s to be unprepared", host) tCtx.Eventually(func(ktesting.TContext) []app.ClaimID { return plugin.GetPreparedResources() }).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host) } diff --git a/test/e2e/dra/utils/deploy.go b/test/e2e/dra/utils/deploy.go index becde2315f8..5e0b16bb777 100644 --- a/test/e2e/dra/utils/deploy.go +++ b/test/e2e/dra/utils/deploy.go @@ -1038,6 +1038,7 @@ func (d *Driver) TearDown(tCtx ktesting.TContext) { // // Only use this in tests where kubelet support for DRA is guaranteed. func (d *Driver) IsGone(tCtx ktesting.TContext) { + tCtx.Helper() tCtx.Logf("Waiting for ResourceSlices of driver %s to be removed...", d.Name) tCtx.Eventually(d.NewGetSlices()).WithTimeout(2 * time.Minute).Should(gomega.HaveField("Items", gomega.BeEmpty())) } diff --git a/test/e2e_dra/devicetaints_test.go b/test/e2e_dra/devicetaints_test.go new file mode 100644 index 00000000000..503337cd2b7 --- /dev/null +++ b/test/e2e_dra/devicetaints_test.go @@ -0,0 +1,137 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2edra + +import ( + "time" + + "github.com/onsi/gomega" + resourceapi "k8s.io/api/resource/v1" + resourcealpha "k8s.io/api/resource/v1alpha3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + drautils "k8s.io/kubernetes/test/e2e/dra/utils" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/utils/ktesting" + "k8s.io/utils/ptr" +) + +// deviceTaints checks that: +// - A pod which gets scheduled on the previous release because of a toleration is kept running after an upgrade. +// - A DeviceTaintRule created to evict the pod before a downgrade prevents pod scheduling after a downgrade. +func deviceTaints(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { + namespace := tCtx.Namespace() + taintKey := "devicetaints" + taintValueFromSlice := "from-slice" + taintValueFromRule := "from-rule" + taintedDevice := "tainted-device" + + // We need additional devices which are only used by this test. + // We achieve that with cluster-scoped devices that start out with + // a taint. + slice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "devicetaints", + }, + Spec: resourceapi.ResourceSliceSpec{ + Driver: b.Driver.Name, + Pool: resourceapi.ResourcePool{ + Name: "devicetaints", + ResourceSliceCount: 1, + }, + AllNodes: ptr.To(true), + Devices: []resourceapi.Device{{ + Name: taintedDevice, + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "example.com/type": { + StringValue: ptr.To("devicetaints"), + }, + }, + Taints: []resourceapi.DeviceTaint{{ + Key: taintKey, + Value: taintValueFromSlice, + Effect: resourceapi.DeviceTaintEffectNoSchedule, + }}, + }}, + }, + } + _, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{}) + tCtx.ExpectNoError(err) + + tCtx.Log("The pod wants exactly the tainted device -> not schedulable.") + claim := b.ExternalClaim() + pod := b.PodExternal() + pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name + claim.Spec.Devices.Requests[0].Exactly.Selectors = []resourceapi.DeviceSelector{{ + CEL: &resourceapi.CELDeviceSelector{ + Expression: `device.attributes["example.com"].?type.orValue("") == "devicetaints"`, + }, + }} + b.Create(tCtx, claim, pod) + tCtx.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(tCtx, tCtx.Client(), pod.Name, namespace)) + + tCtx.Log("Adding a toleration makes the pod schedulable.") + claim.Spec.Devices.Requests[0].Exactly.Tolerations = []resourceapi.DeviceToleration{{ + Key: taintKey, + Value: taintValueFromSlice, + Effect: resourceapi.DeviceTaintEffectNoSchedule, + }} + tCtx.ExpectNoError(tCtx.Client().ResourceV1().ResourceClaims(namespace).Delete(tCtx, claim.Name, metav1.DeleteOptions{})) + _, err = tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{}) + tCtx.ExpectNoError(err) + b.TestPod(tCtx, pod) + + return func(tCtx ktesting.TContext) downgradedTestFunc { + tCtx.Log("Pod running consistently after upgrade.") + tCtx.Consistently(func(tCtx ktesting.TContext) error { + return e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod) + }).WithTimeout(30 * time.Second).WithPolling(5 * time.Second).Should(gomega.Succeed()) + + tCtx.Logf("Evict pod through DeviceTaintRule.") + rule := &resourcealpha.DeviceTaintRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "device-taint-rule", + }, + Spec: resourcealpha.DeviceTaintRuleSpec{ + DeviceSelector: &resourcealpha.DeviceTaintSelector{ + Driver: &b.Driver.Name, + Pool: &slice.Spec.Pool.Name, + Device: &taintedDevice, + }, + Taint: resourcealpha.DeviceTaint{ + Key: taintKey, + Value: taintValueFromRule, + Effect: resourcealpha.DeviceTaintEffectNoExecute, + }, + }, + } + _, err := tCtx.Client().ResourceV1alpha3().DeviceTaintRules().Create(tCtx, rule, metav1.CreateOptions{}) + tCtx.ExpectNoError(err) + tCtx.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(tCtx, tCtx.Client(), pod.Name, namespace, 5*time.Minute)) + + return func(tCtx ktesting.TContext) { + tCtx.Log("DeviceTaintRule still in effect.") + b.Create(tCtx, pod) + tCtx.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(tCtx, tCtx.Client(), pod.Name, namespace)) + + // We must clean up manually, otherwise the code which checks for ResourceSlice deletion after + // driver removal gets stuck waiting for the removal of special ResourceSlice. + // This cannot be scheduled via tCtx.Cleanup after creating it because then it would be removed + // after the first sub-test. + tCtx.ExpectNoError(tCtx.Client().ResourceV1().ResourceSlices().Delete(tCtx, slice.Name, metav1.DeleteOptions{})) + } + } +} diff --git a/test/e2e_dra/upgradedowngrade_test.go b/test/e2e_dra/upgradedowngrade_test.go index b53c06090f5..2abc6e77893 100644 --- a/test/e2e_dra/upgradedowngrade_test.go +++ b/test/e2e_dra/upgradedowngrade_test.go @@ -66,7 +66,8 @@ func init() { // sub-test. That function then returns the next piece of code, which then // returns the final code. Each callback function is executed as a sub-test. // The builder is configured to not delete objects when that sub-test ends, -// so objects persist until the entire test is done. +// so objects persist until the entire test is done. The same DRA driver +// is used for all sub-tests. // // Each sub-test must be self-contained. They intentionally run in a random // order. However, they share the same cluster and the 8 devices which are @@ -74,6 +75,7 @@ func init() { var subTests = map[string]initialTestFunc{ "core DRA": coreDRA, "ResourceClaim device status": resourceClaimDeviceStatus, + "DeviceTaints": deviceTaints, } type initialTestFunc func(tCtx ktesting.TContext, builder *drautils.Builder) upgradedTestFunc @@ -210,8 +212,8 @@ func testUpgradeDowngrade(tCtx ktesting.TContext) { tCtx.Step(fmt.Sprintf("bring up v%d.%d", major, previousMinor), func(tCtx ktesting.TContext) { cluster = localupcluster.New(tCtx) localUpClusterEnv := map[string]string{ - "RUNTIME_CONFIG": "resource.k8s.io/v1beta1,resource.k8s.io/v1beta2", - "FEATURE_GATES": "DynamicResourceAllocation=true", + "RUNTIME_CONFIG": "resource.k8s.io/v1beta1,resource.k8s.io/v1beta2,resource.k8s.io/v1alpha3", + "FEATURE_GATES": "DynamicResourceAllocation=true,DRADeviceTaintRules=true,DRADeviceTaints=true", // *not* needed because driver will run in "local filesystem" mode (= driver.IsLocal): "ALLOW_PRIVILEGED": "1", } cluster.Start(tCtx, binDir, localUpClusterEnv) @@ -247,6 +249,7 @@ func testUpgradeDowngrade(tCtx ktesting.TContext) { }) } }) + numSlices := len(driver.NewGetSlices()(tCtx).Items) // We could split this up into first updating the apiserver, then control plane components, then restarting kubelet. // For the purpose of this test here we we primarily care about full before/after comparisons, so not done yet. @@ -255,7 +258,7 @@ func testUpgradeDowngrade(tCtx ktesting.TContext) { // The kubelet wipes all ResourceSlices on a restart because it doesn't know which drivers were running. // Wait for the ResourceSlice controller in the driver to notice and recreate the ResourceSlices. - tCtx.WithStep("wait for ResourceSlices").Eventually(driver.NewGetSlices()).WithTimeout(5 * time.Minute).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames)))) + tCtx.WithStep("wait for ResourceSlices").Eventually(driver.NewGetSlices()).WithTimeout(5 * time.Minute).Should(gomega.HaveField("Items", gomega.HaveLen(numSlices))) downgradedTestFuncs := make(map[string]downgradedTestFunc, len(subTests)) tCtx.Run("after-cluster-upgrade", func(tCtx ktesting.TContext) { From ff551f387f899dd77534f704f388fa6c6853ddf1 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 11 Feb 2026 12:50:07 +0100 Subject: [PATCH 2/2] DRA E2E: disarm footgun in PodExternal The implicit matching of the ResourceClaim name to ExternalClaim was convenient (no need to specify the parameter) but did go wrong in integration testing where there are multiple calls to ExternalClaim. --- test/e2e/dra/dra.go | 80 ++++++++++------------ test/e2e/dra/utils/builder.go | 17 ++--- test/e2e_dra/coredra_test.go | 4 +- test/e2e_dra/devicetaints_test.go | 3 +- test/integration/dra/resourceclaim_test.go | 7 +- 5 files changed, 51 insertions(+), 60 deletions(-) diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index c27e562e29a..3666fad08fc 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -272,7 +272,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.It("must not run a pod if a claim is not ready", func(ctx context.Context) { claim := b.ExternalClaim() b.Create(f.TContext(ctx), claim) - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) // This bypasses scheduling and therefore the pod gets // to run on the node although the claim is not ready. @@ -295,7 +295,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.It("must unprepare resources for force-deleted pod", func(ctx context.Context) { claim := b.ExternalClaim() - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) zero := int64(0) pod.Spec.TerminationGracePeriodSeconds = &zero @@ -361,7 +361,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { claimForContainer1.Spec.Devices.Config[1].Opaque.Parameters.Raw = []byte(`{"container1_config1":"true"}`) claimForContainer1.Spec.Devices.Config[2].Opaque.Parameters.Raw = []byte(`{"container1_config2":"true"}`) - pod := b.PodExternal() + pod := b.PodExternal("") pod.Spec.ResourceClaims = []v1.PodResourceClaim{ { Name: "all", @@ -445,7 +445,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // that the race goes bad (old pod pending shutdown when // new one arrives) and always schedules to the same node. claim := b.ExternalClaim() - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) node := nodes.NodeNames[0] pod.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": node} oldClaim := b.Create(f.TContext(ctx), claim, pod)[0].(*resourceapi.ResourceClaim) @@ -616,8 +616,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) ginkgo.It("supports init containers with external claims", func(ctx context.Context) { - pod := b.PodExternal() claim := b.ExternalClaim() + pod := b.PodExternal(claim.Name) pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]} pod.Spec.InitContainers[0].Name += "-init" // This must succeed for the pod to start. @@ -628,8 +628,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) ginkgo.It("removes reservation from claim when pod is done", func(ctx context.Context) { - pod := b.PodExternal() claim := b.ExternalClaim() + pod := b.PodExternal(claim.Name) pod.Spec.Containers[0].Command = []string{"true"} b.Create(f.TContext(ctx), claim, pod) @@ -680,7 +680,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b := drautils.NewBuilderNow(tCtx, driver) claim := b.ExternalClaim() - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) // Cannot run pod, no devices. @@ -725,7 +725,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Build behaves the same for both driver instances. b := drautils.NewBuilderNow(tCtx, oldDriver) claim := b.ExternalClaim() - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) @@ -768,7 +768,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Build behaves the same for both driver instances. b := drautils.NewBuilderNow(tCtx, oldDriver) claim := b.ExternalClaim() - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) @@ -814,7 +814,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Build behaves the same for both driver instances. b := drautils.NewBuilderNow(tCtx, oldDriver) claim := b.ExternalClaim() - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) @@ -901,10 +901,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.It("supports sharing a claim concurrently", func(ctx context.Context) { tCtx := f.TContext(ctx) var objects []klog.KMetadata - objects = append(objects, b.ExternalClaim()) + claim := b.ExternalClaim() + objects = append(objects, claim) pods := make([]*v1.Pod, numPods) for i := 0; i < numPods; i++ { - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) pods[i] = pod objects = append(objects, pod) } @@ -1015,18 +1016,18 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.It("supports simple pod referencing external resource claim", func(ctx context.Context) { tCtx := f.TContext(ctx) - pod := b.PodExternal() claim := b.ExternalClaim() + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) }) ginkgo.It("supports external claim referenced by multiple pods", func(ctx context.Context) { tCtx := f.TContext(ctx) - pod1 := b.PodExternal() - pod2 := b.PodExternal() - pod3 := b.PodExternal() claim := b.ExternalClaim() + pod1 := b.PodExternal(claim.Name) + pod2 := b.PodExternal(claim.Name) + pod3 := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod1, pod2, pod3) for _, pod := range []*v1.Pod{pod1, pod2, pod3} { @@ -1036,10 +1037,10 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.It("supports external claim referenced by multiple containers of multiple pods", func(ctx context.Context) { tCtx := f.TContext(ctx) - pod1 := b.PodExternalMultiple() - pod2 := b.PodExternalMultiple() - pod3 := b.PodExternalMultiple() claim := b.ExternalClaim() + pod1 := b.PodExternalMultiple(claim.Name) + pod2 := b.PodExternalMultiple(claim.Name) + pod3 := b.PodExternalMultiple(claim.Name) b.Create(tCtx, claim, pod1, pod2, pod3) for _, pod := range []*v1.Pod{pod1, pod2, pod3} { @@ -1061,8 +1062,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.It("must deallocate after use", func(ctx context.Context) { tCtx := f.TContext(ctx) - pod := b.PodExternal() claim := b.ExternalClaim() + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) { @@ -1086,11 +1087,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { tCtx := f.TContext(ctx) extendedResourceName := deployDevicePlugin(tCtx, f, nodes.NodeNames[0:1], false) - pod := b.PodExternal() + claim := b.ExternalClaim() + pod := b.PodExternal(claim.Name) resources := v1.ResourceList{extendedResourceName: resource.MustParse("1")} pod.Spec.Containers[0].Resources.Requests = resources pod.Spec.Containers[0].Resources.Limits = resources - claim := b.ExternalClaim() b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) }) @@ -1117,13 +1118,12 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b.Create(tCtx, firstClaim, secondClaim) // First pod uses only firstClaim - firstPod := b.PodExternal() + firstPod := b.PodExternal(firstClaim.Name) b.Create(tCtx, firstPod) b.TestPod(tCtx, firstPod) // Second pod uses firstClaim (already prepared) + secondClaim (new) - secondPod := b.PodExternal() - + secondPod := b.PodExternal("") secondPod.Spec.ResourceClaims = []v1.PodResourceClaim{ { Name: "first", @@ -1362,7 +1362,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, }, } - pod := b1.PodExternal() + pod := b1.PodExternal("") podClaimName := "resource-claim" externalClaimName := "external-multiclaim" pod.Spec.ResourceClaims = []v1.PodResourceClaim{ @@ -1453,7 +1453,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, }, } - pod := b1.PodExternal() + pod := b1.PodExternal("") podClaimName := "resource-claim" externalClaimName := "external-multiclaim" pod.Spec.ResourceClaims = []v1.PodResourceClaim{ @@ -1532,7 +1532,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, }, } - pod := b1.PodExternal() + pod := b1.PodExternal("") podClaimName := "resource-claim" externalClaimName := "external-multiclaim" pod.Spec.ResourceClaims = []v1.PodResourceClaim{ @@ -1750,7 +1750,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, }, } - pod := b.PodExternal() + pod := b.PodExternal("") podClaimName := "resource-claim" pod.Spec.ResourceClaims = []v1.PodResourceClaim{ { @@ -1831,7 +1831,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, }, } - pod := b.PodExternal() + pod := b.PodExternal("") podClaimName := "resource-claim" pod.Spec.ResourceClaims = []v1.PodResourceClaim{ { @@ -1904,7 +1904,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // available, there should be sufficient counters left to allocate // a device. claim := b.ExternalClaim() - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) @@ -1912,8 +1912,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // For the second pod, there should not be sufficient counters left, so // it should not succeed. This means the pod should remain in the pending state. claim2 := b.ExternalClaim() - pod2 := b.PodExternal() - pod2.Spec.ResourceClaims[0].ResourceClaimName = &claim2.Name + pod2 := b.PodExternal(claim2.Name) b.Create(tCtx, claim2, pod2) gomega.Consistently(ctx, func(ctx context.Context) error { @@ -1969,8 +1968,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { "memory": resource.MustParse("4Gi"), }, } - pod := b.PodExternal() - pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) @@ -1981,8 +1979,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { "memory": resource.MustParse("8Gi"), }, } - pod2 := b.PodExternal() - pod2.Spec.ResourceClaims[0].ResourceClaimName = &claim2.Name + pod2 := b.PodExternal(claim2.Name) b.Create(tCtx, claim2, pod2) // The third pod should be able to use the rest 4Gi of the device. @@ -1992,8 +1989,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { "memory": resource.MustParse("4Gi"), }, } - pod3 := b.PodExternal() - pod3.Spec.ResourceClaims[0].ResourceClaimName = &claim3.Name + pod3 := b.PodExternal(claim3.Name) b.Create(tCtx, claim3, pod3) b.TestPod(tCtx, pod3) @@ -2298,7 +2294,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...) claim := b.ExternalClaim() - pod2 := b.PodExternal() + pod2 := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod2) b.TestPod(tCtx, pod2) @@ -2832,8 +2828,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { f.It("must be possible for the driver to update the ResourceClaim.Status.Devices once allocated", f.WithFeatureGate(features.DRAResourceClaimDeviceStatus), func(ctx context.Context) { tCtx := f.TContext(ctx) - pod := b.PodExternal() claim := b.ExternalClaim() + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) // Waits for the ResourceClaim to be allocated and the pod to be scheduled. @@ -3073,7 +3069,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { claim1b := b1.ExternalClaim() claim2 := b2.ExternalClaim() claim2b := b2.ExternalClaim() - pod := b1.PodExternal() + pod := b1.PodExternal(claim1.Name) for i, claim := range []*resourceapi.ResourceClaim{claim1b, claim2, claim2b} { pod.Spec.ResourceClaims = append(pod.Spec.ResourceClaims, v1.PodResourceClaim{ diff --git a/test/e2e/dra/utils/builder.go b/test/e2e/dra/utils/builder.go index 82c6cabff91..3404687b49d 100644 --- a/test/e2e/dra/utils/builder.go +++ b/test/e2e/dra/utils/builder.go @@ -313,16 +313,11 @@ func (b *Builder) PodInlineMultiple() (*v1.Pod, *resourceapi.ResourceClaimTempla return pod, template } -// PodExternal adds a pod that references external resource claim with default class name and parameters. -// -// Note that this references *the initial* result of ExternalClaim. When generating multiple such -// external claims, pod.Spec.ResourceClaims[0].ResourceClaimName must be adapted by the caller, -// if desired. -func (b *Builder) PodExternal() *v1.Pod { +// PodExternal adds a pod that references the named resource claim. +func (b *Builder) PodExternal(externalClaimName string) *v1.Pod { pod := b.Pod() pod.Spec.Containers[0].Name = "with-resource" podClaimName := "resource-claim" - externalClaimName := "external-claim" + b.Driver.NameSuffix pod.Spec.ResourceClaims = []v1.PodResourceClaim{ { Name: podClaimName, @@ -333,9 +328,9 @@ func (b *Builder) PodExternal() *v1.Pod { return pod } -// podShared returns a pod with 3 containers that reference external resource claim with default class name and parameters. -func (b *Builder) PodExternalMultiple() *v1.Pod { - pod := b.PodExternal() +// podShared returns a pod with 3 containers that reference the named external resource claim. +func (b *Builder) PodExternalMultiple(externalClaimName string) *v1.Pod { + pod := b.PodExternal(externalClaimName) pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy(), *pod.Spec.Containers[0].DeepCopy()) pod.Spec.Containers[1].Name += "-1" pod.Spec.Containers[2].Name += "-2" @@ -520,7 +515,7 @@ func (b *Builder) tearDown(tCtx ktesting.TContext) { } tCtx.Logf("Deleting %T %s", &pod, klog.KObj(&pod)) options := metav1.DeleteOptions{} - if !b.driver.WithRealNodes { + if !b.Driver.WithRealNodes { // Force-delete, no kubelet. options.GracePeriodSeconds = ptr.To(int64(0)) } diff --git a/test/e2e_dra/coredra_test.go b/test/e2e_dra/coredra_test.go index 293014a03bf..2bc1d483d7d 100644 --- a/test/e2e_dra/coredra_test.go +++ b/test/e2e_dra/coredra_test.go @@ -31,7 +31,7 @@ import ( func coreDRA(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { namespace := tCtx.Namespace() claim := b.ExternalClaim() - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) @@ -43,7 +43,7 @@ func coreDRA(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { // Create another claim and pod, this time using the latest Kubernetes. claim = b.ExternalClaim() - pod = b.PodExternal() + pod = b.PodExternal(claim.Name) pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name b.Create(tCtx, claim, pod) b.TestPod(tCtx, pod) diff --git a/test/e2e_dra/devicetaints_test.go b/test/e2e_dra/devicetaints_test.go index 503337cd2b7..9dc0d55c050 100644 --- a/test/e2e_dra/devicetaints_test.go +++ b/test/e2e_dra/devicetaints_test.go @@ -73,8 +73,7 @@ func deviceTaints(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc tCtx.Log("The pod wants exactly the tainted device -> not schedulable.") claim := b.ExternalClaim() - pod := b.PodExternal() - pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name + pod := b.PodExternal(claim.Name) claim.Spec.Devices.Requests[0].Exactly.Selectors = []resourceapi.DeviceSelector{{ CEL: &resourceapi.CELDeviceSelector{ Expression: `device.attributes["example.com"].?type.orValue("") == "devicetaints"`, diff --git a/test/integration/dra/resourceclaim_test.go b/test/integration/dra/resourceclaim_test.go index a1838189b73..843a9225d1b 100644 --- a/test/integration/dra/resourceclaim_test.go +++ b/test/integration/dra/resourceclaim_test.go @@ -57,7 +57,8 @@ func testShareResourceClaimSequentially(tCtx ktesting.TContext) { b := drautils.NewBuilderNow(tCtx, driver) var objects []klog.KMetadata - objects = append(objects, b.ExternalClaim()) + claim := b.ExternalClaim() + objects = append(objects, claim) // This test used to test usage of the claim by one pod // at a time. After removing the "not sharable" @@ -67,7 +68,7 @@ func testShareResourceClaimSequentially(tCtx ktesting.TContext) { tCtx.Logf("Creating %d pods sharing the same claim", numMaxPods) pods := make([]*v1.Pod, numMaxPods) for i := range numMaxPods { - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) pods[i] = pod objects = append(objects, pod) } @@ -126,7 +127,7 @@ func testShareResourceClaimSequentially(tCtx ktesting.TContext) { morePods := make([]*v1.Pod, numMorePods) objects = nil for i := range numMorePods { - pod := b.PodExternal() + pod := b.PodExternal(claim.Name) morePods[i] = pod objects = append(objects, pod) }