From 642f4798462feb7cb4c7045fde103bca1eca8e23 Mon Sep 17 00:00:00 2001 From: Sai Ramesh Vanka Date: Thu, 12 Feb 2026 20:29:02 +0530 Subject: [PATCH 1/3] Add upgrade/downgrade e2e tests for the DRAExtendedResources feature - Implement e2e tests based on the scenarios described in the PR kubernetes/enhancements#5751 that are required to graduate the DRAExtendedResources feature to Beta Signed-off-by: Sai Ramesh Vanka --- test/e2e_dra/extendedresources_test.go | 348 +++++++++++++++++++++++++ test/e2e_dra/upgradedowngrade_test.go | 3 +- 2 files changed, 350 insertions(+), 1 deletion(-) create mode 100644 test/e2e_dra/extendedresources_test.go diff --git a/test/e2e_dra/extendedresources_test.go b/test/e2e_dra/extendedresources_test.go new file mode 100644 index 00000000000..63562130792 --- /dev/null +++ b/test/e2e_dra/extendedresources_test.go @@ -0,0 +1,348 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2edra + +import ( + "fmt" + "time" + + "github.com/onsi/gomega" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + resourceapi "k8s.io/api/resource/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + drautils "k8s.io/kubernetes/test/e2e/dra/utils" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/utils/ktesting" +) + +// extendedResourcesUpgradeDowngrade tests the DRAExtendedResources feature during upgrade/downgrade scenarios. +// This test verifies that: +// 1. DeviceClass with explicit extendedResourceName field (custom resource name) +// 2. DeviceClass with implicit extendedResourceName (using default format) +// 3. Pods requesting extended resources are scheduled and allocated devices +// 4. The scheduler creates special ResourceClaims for extended resource requests +// 5. Pod status contains extendedResourceClaimStatus mapping +// 6. Feature works correctly across upgrade/downgrade cycles +func extendedResourcesUpgradeDowngrade(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { + namespace := tCtx.Namespace() + + // Test 1: Explicit extended resource name + // DeviceClass with extendedResourceName field explicitly set to a custom value + b.UseExtendedResourceName = true + deviceClass1 := b.Class(drautils.SingletonIndex) + deviceClass1, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, deviceClass1, metav1.CreateOptions{}) + tCtx.ExpectNoError(err, "create DeviceClass with explicit extendedResourceName") + + // Check if the ExtendedResourceName field is supported (may be nil in older versions) + if deviceClass1.Spec.ExtendedResourceName == nil { + tCtx.Logf("ExtendedResourceName feature not supported in this version, skipping extended resources test") + // Return no-op functions for upgrade and downgrade phases + return func(tCtx ktesting.TContext) downgradedTestFunc { + return func(tCtx ktesting.TContext) { + // Cleanup the DeviceClass we created + tCtx.Eventually(func(tCtx ktesting.TContext) error { + return tCtx.Client().ResourceV1().DeviceClasses().Delete(tCtx, deviceClass1.Name, metav1.DeleteOptions{}) + }).Should(gomega.Succeed(), "delete DeviceClass") + } + } + } + + tCtx.Logf("Created DeviceClass %q with explicit extendedResourceName: %s", deviceClass1.Name, *deviceClass1.Spec.ExtendedResourceName) + + // Create a pod that requests the extended resource via container.resources.requests + // The scheduler should automatically create a ResourceClaim for this pod + pod1 := b.Pod() + pod1.Name = "extended-resource-pod-1" + resourceName1 := v1.ResourceName(*deviceClass1.Spec.ExtendedResourceName) + pod1.Spec.Containers[0].Resources.Requests = v1.ResourceList{ + resourceName1: resource.MustParse("1"), + } + pod1.Spec.Containers[0].Resources.Limits = v1.ResourceList{ + resourceName1: resource.MustParse("1"), + } + + pod1, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod1, metav1.CreateOptions{}) + tCtx.ExpectNoError(err, "create pod requesting explicit extended resource") + tCtx.Logf("Created pod %q requesting explicit extended resource %q", pod1.Name, resourceName1) + + // Wait for pod to be scheduled and running + tCtx.ExpectNoError(e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod1), "wait for pod to run") + + // Verify the scheduler created a special ResourceClaim for this pod + var specialClaim *resourceapi.ResourceClaim + tCtx.Eventually(func(tCtx ktesting.TContext) bool { + // The special claim should be named after the pod and extended resource + claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) + if err != nil { + tCtx.Logf("Error listing claims: %v", err) + return false + } + for _, claim := range claims.Items { + // Special claims created by scheduler have owner reference to the pod + for _, ownerRef := range claim.OwnerReferences { + if ownerRef.UID == pod1.UID && ownerRef.Kind == "Pod" { + specialClaim = &claim + return true + } + } + } + return false + }).Should(gomega.BeTrueBecause("scheduler should create special ResourceClaim")) + + tCtx.Logf("Found special ResourceClaim %q created by scheduler for pod %q", specialClaim.Name, pod1.Name) + + // Verify the special claim is allocated + require.NotNil(tCtx, specialClaim.Status.Allocation, "special ResourceClaim should be allocated") + tCtx.Logf("Special ResourceClaim %q is allocated", specialClaim.Name) + + // Get updated pod to check status + pod1, err = tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod1.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod to check status") + + // Verify pod.status.extendedResourceClaimStatus contains the mapping + require.NotNil(tCtx, pod1.Status.ExtendedResourceClaimStatus, "pod.status.extendedResourceClaimStatus should not be nil") + + // Verify the ResourceClaimName matches the special claim + require.Equal(tCtx, specialClaim.Name, pod1.Status.ExtendedResourceClaimStatus.ResourceClaimName, + "extendedResourceClaimStatus.ResourceClaimName should match special claim") + tCtx.Logf("Verified extendedResourceClaimStatus.ResourceClaimName: %s", + pod1.Status.ExtendedResourceClaimStatus.ResourceClaimName) + + // Verify request mappings contain the extended resource + foundMapping := false + for _, reqMapping := range pod1.Status.ExtendedResourceClaimStatus.RequestMappings { + if reqMapping.ResourceName == string(resourceName1) { + foundMapping = true + tCtx.Logf("Found request mapping: container=%s, resource=%s, request=%s", + reqMapping.ContainerName, reqMapping.ResourceName, reqMapping.RequestName) + break + } + } + require.True(tCtx, foundMapping, "extendedResourceClaimStatus.RequestMappings should contain %s", resourceName1) + + // Test 2: Implicit extended resource name + // Create DeviceClass WITHOUT setting extendedResourceName field + // This tests the implicit naming: deviceclass.resource.kubernetes.io/ + // Use index 1 to avoid conflict with the default DeviceClass created by builder.setUp() which uses index 0 + b.UseExtendedResourceName = false + deviceClass2 := b.Class(1) + deviceClass2, err = tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, deviceClass2, metav1.CreateOptions{}) + tCtx.ExpectNoError(err, "create DeviceClass for implicit extended resource") + tCtx.Logf("Created DeviceClass %q for implicit extended resource testing", deviceClass2.Name) + + // The implicit extended resource name format + resourceName2 := v1.ResourceName(fmt.Sprintf("deviceclass.resource.kubernetes.io/%s", deviceClass2.Name)) + tCtx.Logf("Using implicit extended resource name: %s", resourceName2) + + // Create a pod that requests the extended resource + pod3 := b.Pod() + pod3.Name = "extended-resource-pod-2" + pod3.Spec.Containers[0].Resources.Requests = v1.ResourceList{ + resourceName2: resource.MustParse("1"), + } + pod3.Spec.Containers[0].Resources.Limits = v1.ResourceList{ + resourceName2: resource.MustParse("1"), + } + + pod3, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod3, metav1.CreateOptions{}) + tCtx.ExpectNoError(err, "create pod requesting implicit extended resource") + tCtx.Logf("Created pod %q requesting implicit extended resource %q", pod3.Name, resourceName2) + + // Wait for pod to be scheduled and running + tCtx.ExpectNoError(e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod3), "wait for pod to run") + tCtx.Logf("Pod %q with implicit extended resource is running", pod3.Name) + + // Verify scheduler created special ResourceClaim for implicit resource + var claim3 *resourceapi.ResourceClaim + tCtx.Eventually(func(tCtx ktesting.TContext) bool { + claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) + if err != nil { + tCtx.Logf("Error listing claims: %v", err) + return false + } + for _, claim := range claims.Items { + for _, ownerRef := range claim.OwnerReferences { + if ownerRef.UID == pod3.UID && ownerRef.Kind == "Pod" { + claim3 = &claim + return true + } + } + } + return false + }).Should(gomega.BeTrueBecause("scheduler should create special ResourceClaim for implicit resource")) + + tCtx.Logf("Found special ResourceClaim %q for implicit extended resource", claim3.Name) + + return func(tCtx ktesting.TContext) downgradedTestFunc { + // After upgrade: feature is still enabled + // Verify existing pods still run correctly + pod1, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod1.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod1 after upgrade") + require.Equal(tCtx, v1.PodRunning, pod1.Status.Phase, "pod1 should still be running after upgrade") + tCtx.Logf("Verified pod %q (explicit resource) still running after upgrade", pod1.Name) + + pod3, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod3.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod3 after upgrade") + require.Equal(tCtx, v1.PodRunning, pod3.Status.Phase, "pod3 should still be running after upgrade") + tCtx.Logf("Verified pod %q (implicit resource) still running after upgrade", pod3.Name) + + // Create another pod with explicit extended resource request to verify scheduler still works + b.UseExtendedResourceName = true + pod2 := b.Pod() + pod2.Name = "extended-resource-pod-3" + pod2.Spec.Containers[0].Resources.Requests = v1.ResourceList{ + resourceName1: resource.MustParse("1"), + } + pod2.Spec.Containers[0].Resources.Limits = v1.ResourceList{ + resourceName1: resource.MustParse("1"), + } + + pod2, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod2, metav1.CreateOptions{}) + tCtx.ExpectNoError(err, "create pod with explicit resource after upgrade") + tCtx.Logf("Created pod %q with explicit extended resource after upgrade", pod2.Name) + + // Wait for second pod to be scheduled and running + tCtx.ExpectNoError(e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod2), "wait for second pod to run") + tCtx.Logf("Pod %q is running after upgrade", pod2.Name) + + // Verify scheduler created a special claim for the second pod + var claim2 *resourceapi.ResourceClaim + tCtx.Eventually(func(tCtx ktesting.TContext) bool { + claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) + if err != nil { + tCtx.Logf("Error listing claims: %v", err) + return false + } + for _, claim := range claims.Items { + for _, ownerRef := range claim.OwnerReferences { + if ownerRef.UID == pod2.UID && ownerRef.Kind == "Pod" { + claim2 = &claim + return true + } + } + } + return false + }).Should(gomega.BeTrueBecause("scheduler should create special ResourceClaim")) + + tCtx.Logf("Found special ResourceClaim %q for pod %q", claim2.Name, pod2.Name) + + return func(tCtx ktesting.TContext) { + // After downgrade: feature should be disabled or degraded + // Existing pods should continue running (CDI devices remain prepared) + pod1, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod1.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod1 after downgrade") + if pod1.Status.Phase != v1.PodRunning { + tCtx.Logf("Warning: pod %q is in phase %s after downgrade (expected Running)", pod1.Name, pod1.Status.Phase) + } else { + tCtx.Logf("Pod %q still running after downgrade", pod1.Name) + } + + pod2, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod2.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod2 after downgrade") + if pod2.Status.Phase != v1.PodRunning { + tCtx.Logf("Warning: pod %q is in phase %s after downgrade (expected Running)", pod2.Name, pod2.Status.Phase) + } else { + tCtx.Logf("Pod %q still running after downgrade", pod2.Name) + } + + pod3, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod3.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get pod3 after downgrade") + if pod3.Status.Phase != v1.PodRunning { + tCtx.Logf("Warning: pod %q is in phase %s after downgrade (expected Running)", pod3.Name, pod3.Status.Phase) + } else { + tCtx.Logf("Pod %q still running after downgrade", pod3.Name) + } + + // Clean up all pods + tCtx.Logf("Cleaning up pods after downgrade") + tCtx.Eventually(func(tCtx ktesting.TContext) error { + return tCtx.Client().CoreV1().Pods(namespace).Delete(tCtx, pod1.Name, metav1.DeleteOptions{}) + }).Should(gomega.Succeed(), "delete pod1 after downgrade") + tCtx.Eventually(func(tCtx ktesting.TContext) error { + return tCtx.Client().CoreV1().Pods(namespace).Delete(tCtx, pod2.Name, metav1.DeleteOptions{}) + }).Should(gomega.Succeed(), "delete pod2 after downgrade") + tCtx.Eventually(func(tCtx ktesting.TContext) error { + return tCtx.Client().CoreV1().Pods(namespace).Delete(tCtx, pod3.Name, metav1.DeleteOptions{}) + }).Should(gomega.Succeed(), "delete pod3 after downgrade") + + // Wait for pods to be fully deleted + tCtx.Eventually(func(tCtx ktesting.TContext) *v1.Pod { + pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod1.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + tCtx.ExpectNoError(err, "get pod1 during cleanup") + return pod + }).WithTimeout(3*time.Minute).Should(gomega.BeNil(), "pod1 should be deleted") + + tCtx.Eventually(func(tCtx ktesting.TContext) *v1.Pod { + pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod2.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + tCtx.ExpectNoError(err, "get pod2 during cleanup") + return pod + }).WithTimeout(3*time.Minute).Should(gomega.BeNil(), "pod2 should be deleted") + + tCtx.Eventually(func(tCtx ktesting.TContext) *v1.Pod { + pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod3.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + tCtx.ExpectNoError(err, "get pod3 during cleanup") + return pod + }).WithTimeout(3*time.Minute).Should(gomega.BeNil(), "pod3 should be deleted") + + tCtx.Logf("Successfully cleaned up all pods") + + // Verify that special ResourceClaims are cleaned up by garbage collector + tCtx.Eventually(func(tCtx ktesting.TContext) int { + claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) + if err != nil { + tCtx.Logf("Error listing claims: %v", err) + return -1 + } + specialClaimCount := 0 + for _, claim := range claims.Items { + for _, ownerRef := range claim.OwnerReferences { + if ownerRef.Kind == "Pod" { + specialClaimCount++ + break + } + } + } + return specialClaimCount + }).WithTimeout(3*time.Minute).Should(gomega.Equal(0), "special ResourceClaims should be garbage collected") + + tCtx.Logf("Verified special ResourceClaims were garbage collected") + + // Clean up both DeviceClasses (explicit and implicit) + tCtx.Eventually(func(tCtx ktesting.TContext) error { + return tCtx.Client().ResourceV1().DeviceClasses().Delete(tCtx, deviceClass1.Name, metav1.DeleteOptions{}) + }).Should(gomega.Succeed(), "delete DeviceClass with explicit extendedResourceName") + + tCtx.Eventually(func(tCtx ktesting.TContext) error { + return tCtx.Client().ResourceV1().DeviceClasses().Delete(tCtx, deviceClass2.Name, metav1.DeleteOptions{}) + }).Should(gomega.Succeed(), "delete DeviceClass with implicit extendedResourceName") + + tCtx.Logf("Test completed successfully") + } + } +} diff --git a/test/e2e_dra/upgradedowngrade_test.go b/test/e2e_dra/upgradedowngrade_test.go index 2abc6e77893..664bfe17b84 100644 --- a/test/e2e_dra/upgradedowngrade_test.go +++ b/test/e2e_dra/upgradedowngrade_test.go @@ -76,6 +76,7 @@ var subTests = map[string]initialTestFunc{ "core DRA": coreDRA, "ResourceClaim device status": resourceClaimDeviceStatus, "DeviceTaints": deviceTaints, + "DRA extended resources": extendedResourcesUpgradeDowngrade, } type initialTestFunc func(tCtx ktesting.TContext, builder *drautils.Builder) upgradedTestFunc @@ -213,7 +214,7 @@ func testUpgradeDowngrade(tCtx ktesting.TContext) { cluster = localupcluster.New(tCtx) localUpClusterEnv := map[string]string{ "RUNTIME_CONFIG": "resource.k8s.io/v1beta1,resource.k8s.io/v1beta2,resource.k8s.io/v1alpha3", - "FEATURE_GATES": "DynamicResourceAllocation=true,DRADeviceTaintRules=true,DRADeviceTaints=true", + "FEATURE_GATES": "DynamicResourceAllocation=true,DRADeviceTaintRules=true,DRADeviceTaints=true,DRAExtendedResource=true", // *not* needed because driver will run in "local filesystem" mode (= driver.IsLocal): "ALLOW_PRIVILEGED": "1", } cluster.Start(tCtx, binDir, localUpClusterEnv) From 00798373548b2cdf092709adcd00365b2aae43be Mon Sep 17 00:00:00 2001 From: Sai Ramesh Vanka Date: Thu, 19 Feb 2026 18:04:37 +0530 Subject: [PATCH 2/3] Address review comments Signed-off-by: Sai Ramesh Vanka --- test/e2e_dra/extendedresources_test.go | 417 ++++++++++--------------- test/e2e_dra/upgradedowngrade_test.go | 2 +- 2 files changed, 172 insertions(+), 247 deletions(-) diff --git a/test/e2e_dra/extendedresources_test.go b/test/e2e_dra/extendedresources_test.go index 63562130792..574629d6b63 100644 --- a/test/e2e_dra/extendedresources_test.go +++ b/test/e2e_dra/extendedresources_test.go @@ -18,77 +18,43 @@ package e2edra import ( "fmt" + "slices" "time" "github.com/onsi/gomega" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + resourceinternal "k8s.io/kubernetes/pkg/apis/resource" drautils "k8s.io/kubernetes/test/e2e/dra/utils" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/utils/ktesting" ) -// extendedResourcesUpgradeDowngrade tests the DRAExtendedResources feature during upgrade/downgrade scenarios. -// This test verifies that: -// 1. DeviceClass with explicit extendedResourceName field (custom resource name) -// 2. DeviceClass with implicit extendedResourceName (using default format) -// 3. Pods requesting extended resources are scheduled and allocated devices -// 4. The scheduler creates special ResourceClaims for extended resource requests -// 5. Pod status contains extendedResourceClaimStatus mapping -// 6. Feature works correctly across upgrade/downgrade cycles -func extendedResourcesUpgradeDowngrade(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { - namespace := tCtx.Namespace() - - // Test 1: Explicit extended resource name - // DeviceClass with extendedResourceName field explicitly set to a custom value - b.UseExtendedResourceName = true - deviceClass1 := b.Class(drautils.SingletonIndex) - deviceClass1, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, deviceClass1, metav1.CreateOptions{}) - tCtx.ExpectNoError(err, "create DeviceClass with explicit extendedResourceName") - - // Check if the ExtendedResourceName field is supported (may be nil in older versions) - if deviceClass1.Spec.ExtendedResourceName == nil { - tCtx.Logf("ExtendedResourceName feature not supported in this version, skipping extended resources test") - // Return no-op functions for upgrade and downgrade phases - return func(tCtx ktesting.TContext) downgradedTestFunc { - return func(tCtx ktesting.TContext) { - // Cleanup the DeviceClass we created - tCtx.Eventually(func(tCtx ktesting.TContext) error { - return tCtx.Client().ResourceV1().DeviceClasses().Delete(tCtx, deviceClass1.Name, metav1.DeleteOptions{}) - }).Should(gomega.Succeed(), "delete DeviceClass") - } - } +// createPodWithExtendedResourceAndVerifyClaim creates a pod requesting an extended resource, +// waits for it to run, and verifies the scheduler created a special ResourceClaim for it. +// Returns the created pod and the special ResourceClaim. +func createPodWithExtendedResourceAndVerifyClaim(tCtx ktesting.TContext, b *drautils.Builder, pod *v1.Pod, resourceName v1.ResourceName, namespace string) (*v1.Pod, *resourceapi.ResourceClaim) { + // Set resource requests and limits + resource := v1.ResourceList{ + resourceName: resource.MustParse("1"), } + pod.Spec.Containers[0].Resources.Requests = resource + pod.Spec.Containers[0].Resources.Limits = resource - tCtx.Logf("Created DeviceClass %q with explicit extendedResourceName: %s", deviceClass1.Name, *deviceClass1.Spec.ExtendedResourceName) + // Create the pod using builder + objs := b.Create(tCtx, pod) + createdPod := objs[0].(*v1.Pod) + tCtx.Logf("Created pod %q requesting extended resource %q", createdPod.Name, resourceName) - // Create a pod that requests the extended resource via container.resources.requests - // The scheduler should automatically create a ResourceClaim for this pod - pod1 := b.Pod() - pod1.Name = "extended-resource-pod-1" - resourceName1 := v1.ResourceName(*deviceClass1.Spec.ExtendedResourceName) - pod1.Spec.Containers[0].Resources.Requests = v1.ResourceList{ - resourceName1: resource.MustParse("1"), - } - pod1.Spec.Containers[0].Resources.Limits = v1.ResourceList{ - resourceName1: resource.MustParse("1"), - } - - pod1, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod1, metav1.CreateOptions{}) - tCtx.ExpectNoError(err, "create pod requesting explicit extended resource") - tCtx.Logf("Created pod %q requesting explicit extended resource %q", pod1.Name, resourceName1) - - // Wait for pod to be scheduled and running - tCtx.ExpectNoError(e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod1), "wait for pod to run") + // Wait for pod to be running and validate + b.TestPod(tCtx, createdPod) // Verify the scheduler created a special ResourceClaim for this pod var specialClaim *resourceapi.ResourceClaim tCtx.Eventually(func(tCtx ktesting.TContext) bool { - // The special claim should be named after the pod and extended resource claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) if err != nil { tCtx.Logf("Error listing claims: %v", err) @@ -97,7 +63,7 @@ func extendedResourcesUpgradeDowngrade(tCtx ktesting.TContext, b *drautils.Build for _, claim := range claims.Items { // Special claims created by scheduler have owner reference to the pod for _, ownerRef := range claim.OwnerReferences { - if ownerRef.UID == pod1.UID && ownerRef.Kind == "Pod" { + if ownerRef.UID == createdPod.UID && ownerRef.Kind == "Pod" { specialClaim = &claim return true } @@ -106,213 +72,155 @@ func extendedResourcesUpgradeDowngrade(tCtx ktesting.TContext, b *drautils.Build return false }).Should(gomega.BeTrueBecause("scheduler should create special ResourceClaim")) - tCtx.Logf("Found special ResourceClaim %q created by scheduler for pod %q", specialClaim.Name, pod1.Name) + tCtx.Logf("Found special ResourceClaim %q created by scheduler for pod %q", specialClaim.Name, createdPod.Name) + + return createdPod, specialClaim +} + +// testExtendedResourcePod creates and verifies a pod requesting an extended resource. +// It validates that the scheduler creates a special ResourceClaim and that the pod status +// contains the proper extendedResourceClaimStatus mapping. +func testExtendedResourcePod(tCtx ktesting.TContext, b *drautils.Builder, resourceName v1.ResourceName, podName string) *v1.Pod { + namespace := tCtx.Namespace() + + pod := b.Pod() + pod.Name = podName + + pod, specialClaim := createPodWithExtendedResourceAndVerifyClaim(tCtx, b, pod, resourceName, namespace) // Verify the special claim is allocated require.NotNil(tCtx, specialClaim.Status.Allocation, "special ResourceClaim should be allocated") tCtx.Logf("Special ResourceClaim %q is allocated", specialClaim.Name) // Get updated pod to check status - pod1, err = tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod1.Name, metav1.GetOptions{}) + pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) tCtx.ExpectNoError(err, "get pod to check status") // Verify pod.status.extendedResourceClaimStatus contains the mapping - require.NotNil(tCtx, pod1.Status.ExtendedResourceClaimStatus, "pod.status.extendedResourceClaimStatus should not be nil") + require.NotNil(tCtx, pod.Status.ExtendedResourceClaimStatus, "pod.status.extendedResourceClaimStatus should not be nil") // Verify the ResourceClaimName matches the special claim - require.Equal(tCtx, specialClaim.Name, pod1.Status.ExtendedResourceClaimStatus.ResourceClaimName, + require.Equal(tCtx, specialClaim.Name, pod.Status.ExtendedResourceClaimStatus.ResourceClaimName, "extendedResourceClaimStatus.ResourceClaimName should match special claim") tCtx.Logf("Verified extendedResourceClaimStatus.ResourceClaimName: %s", - pod1.Status.ExtendedResourceClaimStatus.ResourceClaimName) + pod.Status.ExtendedResourceClaimStatus.ResourceClaimName) // Verify request mappings contain the extended resource - foundMapping := false - for _, reqMapping := range pod1.Status.ExtendedResourceClaimStatus.RequestMappings { - if reqMapping.ResourceName == string(resourceName1) { - foundMapping = true + foundMapping := slices.ContainsFunc(pod.Status.ExtendedResourceClaimStatus.RequestMappings, func(reqMapping v1.ContainerExtendedResourceRequest) bool { + if reqMapping.ResourceName == string(resourceName) { tCtx.Logf("Found request mapping: container=%s, resource=%s, request=%s", reqMapping.ContainerName, reqMapping.ResourceName, reqMapping.RequestName) - break - } - } - require.True(tCtx, foundMapping, "extendedResourceClaimStatus.RequestMappings should contain %s", resourceName1) - - // Test 2: Implicit extended resource name - // Create DeviceClass WITHOUT setting extendedResourceName field - // This tests the implicit naming: deviceclass.resource.kubernetes.io/ - // Use index 1 to avoid conflict with the default DeviceClass created by builder.setUp() which uses index 0 - b.UseExtendedResourceName = false - deviceClass2 := b.Class(1) - deviceClass2, err = tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, deviceClass2, metav1.CreateOptions{}) - tCtx.ExpectNoError(err, "create DeviceClass for implicit extended resource") - tCtx.Logf("Created DeviceClass %q for implicit extended resource testing", deviceClass2.Name) - - // The implicit extended resource name format - resourceName2 := v1.ResourceName(fmt.Sprintf("deviceclass.resource.kubernetes.io/%s", deviceClass2.Name)) - tCtx.Logf("Using implicit extended resource name: %s", resourceName2) - - // Create a pod that requests the extended resource - pod3 := b.Pod() - pod3.Name = "extended-resource-pod-2" - pod3.Spec.Containers[0].Resources.Requests = v1.ResourceList{ - resourceName2: resource.MustParse("1"), - } - pod3.Spec.Containers[0].Resources.Limits = v1.ResourceList{ - resourceName2: resource.MustParse("1"), - } - - pod3, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod3, metav1.CreateOptions{}) - tCtx.ExpectNoError(err, "create pod requesting implicit extended resource") - tCtx.Logf("Created pod %q requesting implicit extended resource %q", pod3.Name, resourceName2) - - // Wait for pod to be scheduled and running - tCtx.ExpectNoError(e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod3), "wait for pod to run") - tCtx.Logf("Pod %q with implicit extended resource is running", pod3.Name) - - // Verify scheduler created special ResourceClaim for implicit resource - var claim3 *resourceapi.ResourceClaim - tCtx.Eventually(func(tCtx ktesting.TContext) bool { - claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) - if err != nil { - tCtx.Logf("Error listing claims: %v", err) - return false - } - for _, claim := range claims.Items { - for _, ownerRef := range claim.OwnerReferences { - if ownerRef.UID == pod3.UID && ownerRef.Kind == "Pod" { - claim3 = &claim - return true - } - } + return true } return false - }).Should(gomega.BeTrueBecause("scheduler should create special ResourceClaim for implicit resource")) + }) + require.True(tCtx, foundMapping, "extendedResourceClaimStatus.RequestMappings should contain %s", resourceName) - tCtx.Logf("Found special ResourceClaim %q for implicit extended resource", claim3.Name) + return pod +} + +// verifyPodsRunning verifies that pods are running and logs their status. +// After downgrade, pods may not be running, so we just log warnings instead of failing. +func verifyPodsRunning(tCtx ktesting.TContext, namespace string, podNames []string, resourceType, phase string) { + for _, podName := range podNames { + pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, podName, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get %s pod %s after %s", resourceType, podName, phase) + if pod.Status.Phase != v1.PodRunning { + tCtx.Logf("Warning: pod %q (%s resource) is in phase %s after %s (expected Running)", podName, resourceType, pod.Status.Phase, phase) + } else { + tCtx.Logf("Pod %q (%s resource) still running after %s", podName, resourceType, phase) + } + } +} + +// cleanupPods deletes the specified pods and waits for them to be deleted. +func cleanupPods(tCtx ktesting.TContext, namespace string, podNames []string, resourceType string) { + tCtx.Logf("Cleaning up %s resource pods after downgrade", resourceType) + for _, podName := range podNames { + tCtx.ExpectNoError(e2epod.DeletePodWithWaitByName(tCtx, tCtx.Client(), podName, namespace), "delete %s pod %s", resourceType, podName) + } + tCtx.Logf("Successfully cleaned up %s resource pods", resourceType) +} + +// createDeviceClassForExtendedResource creates a DeviceClass for testing extended resources. +// It sets up the builder state, creates the DeviceClass with a unique name, and returns +// the created DeviceClass and its corresponding resource name. +func createDeviceClassForExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, namespace string, useExplicit bool) (*resourceapi.DeviceClass, v1.ResourceName) { + var classIndex int + var nameSuffix string + + if useExplicit { + b.UseExtendedResourceName = true + classIndex = drautils.SingletonIndex + nameSuffix = "-extended-resource-explicit" + } else { + b.UseExtendedResourceName = false + classIndex = 1 + nameSuffix = "-extended-resource-implicit" + } + + // Create the DeviceClass with a unique name + deviceClass := b.Class(classIndex) + deviceClass.Name = namespace + nameSuffix + createdObjs := b.Create(tCtx, deviceClass) + deviceClass = createdObjs[0].(*resourceapi.DeviceClass) + + // Determine the resource name based on the type + var resourceName v1.ResourceName + if useExplicit { + // Verify ExtendedResourceName is present in the spec when using explicit resource name + // This ensures the API server hasn't stripped the field due to feature gates + require.NotNil(tCtx, deviceClass.Spec.ExtendedResourceName, "DeviceClass.Spec.ExtendedResourceName should be set for explicit resource name") + resourceName = v1.ResourceName(*deviceClass.Spec.ExtendedResourceName) + } else { + resourceName = v1.ResourceName(fmt.Sprintf("%s%s", resourceinternal.ResourceDeviceClassPrefix, deviceClass.Name)) + } + + tCtx.Logf("Created DeviceClass %q with extendedResourceName: %s", deviceClass.Name, resourceName) + + return deviceClass, resourceName +} + +// testExtendedResource tests a single extended resource type through the upgrade/downgrade cycle. +// It creates the DeviceClass, creates pods before upgrade, verifies them after upgrade, +// creates new pods after upgrade, and verifies and cleans up all pods after downgrade. +func testExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, useExplicit bool) upgradedTestFunc { + namespace := tCtx.Namespace() + + // Create DeviceClass and get the resource name + _, resourceName := createDeviceClassForExtendedResource(tCtx, b, namespace, useExplicit) + + // Determine resource type string for logging + resourceType := "implicit" + if useExplicit { + resourceType = "explicit" + } + + // Before upgrade: create and verify pod + podBefore := testExtendedResourcePod(tCtx, b, resourceName, fmt.Sprintf("%s-before-upgrade", resourceType)) + tCtx.Logf("Created pod %q for %s resource before upgrade", podBefore.Name, resourceType) return func(tCtx ktesting.TContext) downgradedTestFunc { - // After upgrade: feature is still enabled - // Verify existing pods still run correctly - pod1, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod1.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get pod1 after upgrade") - require.Equal(tCtx, v1.PodRunning, pod1.Status.Phase, "pod1 should still be running after upgrade") - tCtx.Logf("Verified pod %q (explicit resource) still running after upgrade", pod1.Name) + // After upgrade: verify existing pod still runs correctly + podBefore, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, podBefore.Name, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get %s pod after upgrade", resourceType) + require.Equal(tCtx, v1.PodRunning, podBefore.Status.Phase, "%s pod should still be running after upgrade", resourceType) + tCtx.Logf("Verified pod %q (%s resource) still running after upgrade", podBefore.Name, resourceType) - pod3, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod3.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get pod3 after upgrade") - require.Equal(tCtx, v1.PodRunning, pod3.Status.Phase, "pod3 should still be running after upgrade") - tCtx.Logf("Verified pod %q (implicit resource) still running after upgrade", pod3.Name) - - // Create another pod with explicit extended resource request to verify scheduler still works - b.UseExtendedResourceName = true - pod2 := b.Pod() - pod2.Name = "extended-resource-pod-3" - pod2.Spec.Containers[0].Resources.Requests = v1.ResourceList{ - resourceName1: resource.MustParse("1"), - } - pod2.Spec.Containers[0].Resources.Limits = v1.ResourceList{ - resourceName1: resource.MustParse("1"), - } - - pod2, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod2, metav1.CreateOptions{}) - tCtx.ExpectNoError(err, "create pod with explicit resource after upgrade") - tCtx.Logf("Created pod %q with explicit extended resource after upgrade", pod2.Name) - - // Wait for second pod to be scheduled and running - tCtx.ExpectNoError(e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod2), "wait for second pod to run") - tCtx.Logf("Pod %q is running after upgrade", pod2.Name) - - // Verify scheduler created a special claim for the second pod - var claim2 *resourceapi.ResourceClaim - tCtx.Eventually(func(tCtx ktesting.TContext) bool { - claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) - if err != nil { - tCtx.Logf("Error listing claims: %v", err) - return false - } - for _, claim := range claims.Items { - for _, ownerRef := range claim.OwnerReferences { - if ownerRef.UID == pod2.UID && ownerRef.Kind == "Pod" { - claim2 = &claim - return true - } - } - } - return false - }).Should(gomega.BeTrueBecause("scheduler should create special ResourceClaim")) - - tCtx.Logf("Found special ResourceClaim %q for pod %q", claim2.Name, pod2.Name) + // Create and verify another pod with the same extended resource + podAfter := testExtendedResourcePod(tCtx, b, resourceName, fmt.Sprintf("%s-after-upgrade", resourceType)) + tCtx.Logf("Created pod %q for %s resource after upgrade", podAfter.Name, resourceType) return func(tCtx ktesting.TContext) { - // After downgrade: feature should be disabled or degraded - // Existing pods should continue running (CDI devices remain prepared) - pod1, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod1.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get pod1 after downgrade") - if pod1.Status.Phase != v1.PodRunning { - tCtx.Logf("Warning: pod %q is in phase %s after downgrade (expected Running)", pod1.Name, pod1.Status.Phase) - } else { - tCtx.Logf("Pod %q still running after downgrade", pod1.Name) - } + // After downgrade: verify all pods still run correctly + podNames := []string{podBefore.Name, podAfter.Name} + podUIDs := []string{string(podBefore.UID), string(podAfter.UID)} + verifyPodsRunning(tCtx, namespace, podNames, resourceType, "downgrade") - pod2, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod2.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get pod2 after downgrade") - if pod2.Status.Phase != v1.PodRunning { - tCtx.Logf("Warning: pod %q is in phase %s after downgrade (expected Running)", pod2.Name, pod2.Status.Phase) - } else { - tCtx.Logf("Pod %q still running after downgrade", pod2.Name) - } + // Clean up pods + cleanupPods(tCtx, namespace, podNames, resourceType) - pod3, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod3.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get pod3 after downgrade") - if pod3.Status.Phase != v1.PodRunning { - tCtx.Logf("Warning: pod %q is in phase %s after downgrade (expected Running)", pod3.Name, pod3.Status.Phase) - } else { - tCtx.Logf("Pod %q still running after downgrade", pod3.Name) - } - - // Clean up all pods - tCtx.Logf("Cleaning up pods after downgrade") - tCtx.Eventually(func(tCtx ktesting.TContext) error { - return tCtx.Client().CoreV1().Pods(namespace).Delete(tCtx, pod1.Name, metav1.DeleteOptions{}) - }).Should(gomega.Succeed(), "delete pod1 after downgrade") - tCtx.Eventually(func(tCtx ktesting.TContext) error { - return tCtx.Client().CoreV1().Pods(namespace).Delete(tCtx, pod2.Name, metav1.DeleteOptions{}) - }).Should(gomega.Succeed(), "delete pod2 after downgrade") - tCtx.Eventually(func(tCtx ktesting.TContext) error { - return tCtx.Client().CoreV1().Pods(namespace).Delete(tCtx, pod3.Name, metav1.DeleteOptions{}) - }).Should(gomega.Succeed(), "delete pod3 after downgrade") - - // Wait for pods to be fully deleted - tCtx.Eventually(func(tCtx ktesting.TContext) *v1.Pod { - pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod1.Name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return nil - } - tCtx.ExpectNoError(err, "get pod1 during cleanup") - return pod - }).WithTimeout(3*time.Minute).Should(gomega.BeNil(), "pod1 should be deleted") - - tCtx.Eventually(func(tCtx ktesting.TContext) *v1.Pod { - pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod2.Name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return nil - } - tCtx.ExpectNoError(err, "get pod2 during cleanup") - return pod - }).WithTimeout(3*time.Minute).Should(gomega.BeNil(), "pod2 should be deleted") - - tCtx.Eventually(func(tCtx ktesting.TContext) *v1.Pod { - pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod3.Name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return nil - } - tCtx.ExpectNoError(err, "get pod3 during cleanup") - return pod - }).WithTimeout(3*time.Minute).Should(gomega.BeNil(), "pod3 should be deleted") - - tCtx.Logf("Successfully cleaned up all pods") - - // Verify that special ResourceClaims are cleaned up by garbage collector + // Verify that special ResourceClaims for our pods are cleaned up by garbage collector tCtx.Eventually(func(tCtx ktesting.TContext) int { claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) if err != nil { @@ -322,27 +230,44 @@ func extendedResourcesUpgradeDowngrade(tCtx ktesting.TContext, b *drautils.Build specialClaimCount := 0 for _, claim := range claims.Items { for _, ownerRef := range claim.OwnerReferences { - if ownerRef.Kind == "Pod" { + // Only count claims owned by our specific pods + if ownerRef.Kind == "Pod" && slices.Contains(podUIDs, string(ownerRef.UID)) { specialClaimCount++ break } } } return specialClaimCount - }).WithTimeout(3*time.Minute).Should(gomega.Equal(0), "special ResourceClaims should be garbage collected") + }).WithTimeout(3*time.Minute).Should(gomega.Equal(0), "special ResourceClaims for %s pods should be garbage collected", resourceType) - tCtx.Logf("Verified special ResourceClaims were garbage collected") - - // Clean up both DeviceClasses (explicit and implicit) - tCtx.Eventually(func(tCtx ktesting.TContext) error { - return tCtx.Client().ResourceV1().DeviceClasses().Delete(tCtx, deviceClass1.Name, metav1.DeleteOptions{}) - }).Should(gomega.Succeed(), "delete DeviceClass with explicit extendedResourceName") - - tCtx.Eventually(func(tCtx ktesting.TContext) error { - return tCtx.Client().ResourceV1().DeviceClasses().Delete(tCtx, deviceClass2.Name, metav1.DeleteOptions{}) - }).Should(gomega.Succeed(), "delete DeviceClass with implicit extendedResourceName") - - tCtx.Logf("Test completed successfully") + tCtx.Logf("Verified special ResourceClaims for %s resource were garbage collected", resourceType) + } + } +} + +// extendedResourceUpgradeDowngrade tests the DRAExtendedResources feature during upgrade/downgrade scenarios. +// This test verifies that: +// 1. DeviceClass with explicit extendedResourceName field (custom resource name) +// 2. DeviceClass with implicit extendedResourceName (using default format) +// 3. Pods requesting extended resources are scheduled and allocated devices +// 4. The scheduler creates special ResourceClaims for extended resource requests +// 5. Pod status contains extendedResourceClaimStatus mapping +// 6. Feature works correctly across upgrade/downgrade cycles +func extendedResourceUpgradeDowngrade(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { + // Test both explicit and implicit extended resources through upgrade/downgrade + explicitUpgradedFunc := testExtendedResource(tCtx, b, true) + implicitUpgradedFunc := testExtendedResource(tCtx, b, false) + + return func(tCtx ktesting.TContext) downgradedTestFunc { + explicitDowngradedFunc := explicitUpgradedFunc(tCtx) + implicitDowngradedFunc := implicitUpgradedFunc(tCtx) + + return func(tCtx ktesting.TContext) { + // Run downgrade verification and cleanup for both resource types + explicitDowngradedFunc(tCtx) + implicitDowngradedFunc(tCtx) + + tCtx.Logf("Extended resource upgrade/downgrade test completed successfully") } } } diff --git a/test/e2e_dra/upgradedowngrade_test.go b/test/e2e_dra/upgradedowngrade_test.go index 664bfe17b84..f8b0579aec0 100644 --- a/test/e2e_dra/upgradedowngrade_test.go +++ b/test/e2e_dra/upgradedowngrade_test.go @@ -76,7 +76,7 @@ var subTests = map[string]initialTestFunc{ "core DRA": coreDRA, "ResourceClaim device status": resourceClaimDeviceStatus, "DeviceTaints": deviceTaints, - "DRA extended resources": extendedResourcesUpgradeDowngrade, + "DRAExtendedResource": extendedResourceUpgradeDowngrade, } type initialTestFunc func(tCtx ktesting.TContext, builder *drautils.Builder) upgradedTestFunc From 4bc4b7313ed0d70fea8859a6ce0cfad65f1ce21e Mon Sep 17 00:00:00 2001 From: Sai Ramesh Vanka Date: Wed, 4 Mar 2026 12:48:44 +0530 Subject: [PATCH 3/3] Address review comments - Rebase the code with #137046 changes - Remove the duplication in the DRAExtendedResource upgrade/downgrade test code Signed-off-by: Sai Ramesh Vanka --- test/e2e_dra/extendedresources_test.go | 357 ++++++++++--------------- test/e2e_dra/upgradedowngrade_test.go | 3 +- 2 files changed, 143 insertions(+), 217 deletions(-) diff --git a/test/e2e_dra/extendedresources_test.go b/test/e2e_dra/extendedresources_test.go index 574629d6b63..fb86f4fd303 100644 --- a/test/e2e_dra/extendedresources_test.go +++ b/test/e2e_dra/extendedresources_test.go @@ -22,252 +22,177 @@ import ( "time" "github.com/onsi/gomega" - "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - resourceinternal "k8s.io/kubernetes/pkg/apis/resource" drautils "k8s.io/kubernetes/test/e2e/dra/utils" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/utils/ktesting" ) -// createPodWithExtendedResourceAndVerifyClaim creates a pod requesting an extended resource, -// waits for it to run, and verifies the scheduler created a special ResourceClaim for it. -// Returns the created pod and the special ResourceClaim. -func createPodWithExtendedResourceAndVerifyClaim(tCtx ktesting.TContext, b *drautils.Builder, pod *v1.Pod, resourceName v1.ResourceName, namespace string) (*v1.Pod, *resourceapi.ResourceClaim) { - // Set resource requests and limits - resource := v1.ResourceList{ - resourceName: resource.MustParse("1"), +const ( + resourceTypeExplicit = "explicit" + resourceTypeImplicit = "implicit" +) + +// extendedResourceUpgradeDowngrade tests the DRAExtendedResources feature during upgrade/downgrade scenarios +// for a specific resource type (explicit or implicit). +// This test verifies: +// 1. DeviceClasses can be created with the specified extendedResourceName type +// 2. Pods requesting extended resources are scheduled and allocated devices before upgrade +// 3. Existing pods continue running correctly after upgrade +// 4. New pods with extended resources work correctly after upgrade +// 5. All pods continue running correctly after downgrade +// 6. ResourceClaims are garbage collected when pods are deleted +func extendedResourceUpgradeDowngrade(resourceType string) initialTestFunc { + return func(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { + return testExtendedResource(tCtx, b, resourceType) } - pod.Spec.Containers[0].Resources.Requests = resource - pod.Spec.Containers[0].Resources.Limits = resource - - // Create the pod using builder - objs := b.Create(tCtx, pod) - createdPod := objs[0].(*v1.Pod) - tCtx.Logf("Created pod %q requesting extended resource %q", createdPod.Name, resourceName) - - // Wait for pod to be running and validate - b.TestPod(tCtx, createdPod) - - // Verify the scheduler created a special ResourceClaim for this pod - var specialClaim *resourceapi.ResourceClaim - tCtx.Eventually(func(tCtx ktesting.TContext) bool { - claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) - if err != nil { - tCtx.Logf("Error listing claims: %v", err) - return false - } - for _, claim := range claims.Items { - // Special claims created by scheduler have owner reference to the pod - for _, ownerRef := range claim.OwnerReferences { - if ownerRef.UID == createdPod.UID && ownerRef.Kind == "Pod" { - specialClaim = &claim - return true - } - } - } - return false - }).Should(gomega.BeTrueBecause("scheduler should create special ResourceClaim")) - - tCtx.Logf("Found special ResourceClaim %q created by scheduler for pod %q", specialClaim.Name, createdPod.Name) - - return createdPod, specialClaim } -// testExtendedResourcePod creates and verifies a pod requesting an extended resource. -// It validates that the scheduler creates a special ResourceClaim and that the pod status -// contains the proper extendedResourceClaimStatus mapping. -func testExtendedResourcePod(tCtx ktesting.TContext, b *drautils.Builder, resourceName v1.ResourceName, podName string) *v1.Pod { +// testExtendedResource tests a single extended resource type through the upgrade/downgrade cycle. +// It creates the DeviceClass, creates pods before upgrade, verifies them after upgrade, +// creates new pods after upgrade, and verifies and cleans up all pods after downgrade. +func testExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, resourceType string) upgradedTestFunc { namespace := tCtx.Namespace() + // Create DeviceClass and get the resource name + resourceName := createDeviceClassForExtendedResource(tCtx, b, resourceType) + + // Before upgrade: create and verify pod + podBefore := createPodWithExtendedResource(tCtx, b, fmt.Sprintf("%s-before-upgrade", resourceType), resourceName) + verifyPodRunningWithClaim(tCtx, b, podBefore, resourceName) + + return func(tCtx ktesting.TContext) downgradedTestFunc { + // After upgrade: verify existing pod still runs correctly + verifyPodRunningWithClaim(tCtx, b, podBefore, resourceName) + + // Create and verify another pod with the same extended resource + podAfter := createPodWithExtendedResource(tCtx, b, fmt.Sprintf("%s-after-upgrade", resourceType), resourceName) + verifyPodRunningWithClaim(tCtx, b, podAfter, resourceName) + + return func(tCtx ktesting.TContext) { + // After downgrade: verify all pods still run correctly with their claims + podNames := []string{podBefore.Name, podAfter.Name} + claimNames := make([]string, 0, len(podNames)) + + for _, podName := range podNames { + pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, podName, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get %s pod %s", resourceType, podName) + verifyPodRunningWithClaim(tCtx, b, pod, resourceName) + + // Collect claim name before cleanup + if pod.Status.ExtendedResourceClaimStatus != nil { + claimNames = append(claimNames, pod.Status.ExtendedResourceClaimStatus.ResourceClaimName) + } + } + + // Clean up pods + tCtx.Logf("Cleaning up %s resource pods after downgrade", resourceType) + for _, podName := range podNames { + tCtx.ExpectNoError(e2epod.DeletePodWithWaitByName(tCtx, tCtx.Client(), podName, namespace), "delete %s pod %s", resourceType, podName) + } + tCtx.Logf("Successfully cleaned up %s resource pods", resourceType) + + // Verify that ResourceClaims for our pods are cleaned up by garbage collector + remainingClaims := claimNames + tCtx.Eventually(func(tCtx ktesting.TContext) int { + stillExisting := make([]string, 0, len(remainingClaims)) + for _, claimName := range remainingClaims { + _, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claimName, metav1.GetOptions{}) + if err == nil { + stillExisting = append(stillExisting, claimName) + } else if !apierrors.IsNotFound(err) { + // If we get a real API error (not NotFound), fail immediately + tCtx.ExpectNoError(err, "unexpected error checking ResourceClaim %s", claimName) + } + // If IsNotFound, the claim has been deleted (expected behavior) + } + remainingClaims = stillExisting + return len(remainingClaims) + }).WithTimeout(3*time.Minute).Should(gomega.Equal(0), "ResourceClaims for %s pods should be garbage collected", resourceType) + + tCtx.Logf("Verified ResourceClaims for %s resource were garbage collected", resourceType) + } + } +} + +// createDeviceClassForExtendedResource creates a DeviceClass for testing extended resources. +// It creates the DeviceClass with a unique name, and returns its corresponding resource name. +func createDeviceClassForExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, resourceType string) v1.ResourceName { + var deviceClass *resourceapi.DeviceClass + var resourceName v1.ResourceName + + if resourceType == resourceTypeExplicit { + // Framework automatically names it: b.ClassName() + "-explicit" + resourceName = v1.ResourceName(b.ExtendedResourceName(resourceTypeExplicit)) + deviceClass = b.ClassWithExtendedResource(string(resourceName)) + } else { + // Framework automatically names it + deviceClass = b.Class().DeviceClass + resourceName = v1.ResourceName(fmt.Sprintf("%s%s", resourceapi.ResourceDeviceClassPrefix, deviceClass.Name)) + } + + createdObjs := b.Create(tCtx, deviceClass) + + // Verify ExtendedResourceName is present in the spec when using explicit resource name + if resourceType == resourceTypeExplicit { + deviceClass = createdObjs[0].(*resourceapi.DeviceClass) + tCtx.Expect(deviceClass.Spec.ExtendedResourceName).ToNot(gomega.BeNil(), "DeviceClass.Spec.ExtendedResourceName should be set for explicit resource name") + } + + tCtx.Logf("Created DeviceClass %q with extendedResourceName: %s", deviceClass.Name, resourceName) + + return resourceName +} + +// createPodWithExtendedResource creates a pod with the given name requesting an extended resource. +func createPodWithExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, podName string, resourceName v1.ResourceName) *v1.Pod { pod := b.Pod() pod.Name = podName + resources := v1.ResourceList{resourceName: resource.MustParse("1")} + pod.Spec.Containers[0].Resources.Requests = resources + pod.Spec.Containers[0].Resources.Limits = resources + pod = b.Create(tCtx, pod)[0].(*v1.Pod) + tCtx.Logf("Created pod %q requesting extended resource %q", pod.Name, resourceName) + return pod +} - pod, specialClaim := createPodWithExtendedResourceAndVerifyClaim(tCtx, b, pod, resourceName, namespace) - - // Verify the special claim is allocated - require.NotNil(tCtx, specialClaim.Status.Allocation, "special ResourceClaim should be allocated") - tCtx.Logf("Special ResourceClaim %q is allocated", specialClaim.Name) +// verifyPodRunningWithClaim verifies a pod is running and has a valid ResourceClaim. +func verifyPodRunningWithClaim(tCtx ktesting.TContext, b *drautils.Builder, pod *v1.Pod, resourceName v1.ResourceName) { + namespace := tCtx.Namespace() + b.TestPod(tCtx, pod) + tCtx.Logf("Pod %q is running", pod.Name) // Get updated pod to check status pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) tCtx.ExpectNoError(err, "get pod to check status") // Verify pod.status.extendedResourceClaimStatus contains the mapping - require.NotNil(tCtx, pod.Status.ExtendedResourceClaimStatus, "pod.status.extendedResourceClaimStatus should not be nil") + tCtx.Expect(pod.Status.ExtendedResourceClaimStatus).ToNot(gomega.BeNil(), "pod.status.extendedResourceClaimStatus should not be nil for pod %s", pod.Name) - // Verify the ResourceClaimName matches the special claim - require.Equal(tCtx, specialClaim.Name, pod.Status.ExtendedResourceClaimStatus.ResourceClaimName, - "extendedResourceClaimStatus.ResourceClaimName should match special claim") - tCtx.Logf("Verified extendedResourceClaimStatus.ResourceClaimName: %s", - pod.Status.ExtendedResourceClaimStatus.ResourceClaimName) + // Get the claim name from pod status + claimName := pod.Status.ExtendedResourceClaimStatus.ResourceClaimName + tCtx.Logf("Verified extendedResourceClaimStatus.ResourceClaimName: %s for pod %q", claimName, pod.Name) + + // Fetch the ResourceClaim using the name from pod status + claim, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claimName, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "getting ResourceClaim %s", claimName) + + // Verify the claim is allocated + tCtx.Expect(claim.Status.Allocation).ToNot(gomega.BeNil(), "ResourceClaim %s should be allocated", claim.Name) + tCtx.Logf("ResourceClaim %q is allocated for pod %q", claim.Name, pod.Name) // Verify request mappings contain the extended resource foundMapping := slices.ContainsFunc(pod.Status.ExtendedResourceClaimStatus.RequestMappings, func(reqMapping v1.ContainerExtendedResourceRequest) bool { if reqMapping.ResourceName == string(resourceName) { - tCtx.Logf("Found request mapping: container=%s, resource=%s, request=%s", - reqMapping.ContainerName, reqMapping.ResourceName, reqMapping.RequestName) + tCtx.Logf("Found request mapping for pod %q: container=%s, resource=%s, request=%s", + pod.Name, reqMapping.ContainerName, reqMapping.ResourceName, reqMapping.RequestName) return true } return false }) - require.True(tCtx, foundMapping, "extendedResourceClaimStatus.RequestMappings should contain %s", resourceName) - - return pod -} - -// verifyPodsRunning verifies that pods are running and logs their status. -// After downgrade, pods may not be running, so we just log warnings instead of failing. -func verifyPodsRunning(tCtx ktesting.TContext, namespace string, podNames []string, resourceType, phase string) { - for _, podName := range podNames { - pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, podName, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get %s pod %s after %s", resourceType, podName, phase) - if pod.Status.Phase != v1.PodRunning { - tCtx.Logf("Warning: pod %q (%s resource) is in phase %s after %s (expected Running)", podName, resourceType, pod.Status.Phase, phase) - } else { - tCtx.Logf("Pod %q (%s resource) still running after %s", podName, resourceType, phase) - } - } -} - -// cleanupPods deletes the specified pods and waits for them to be deleted. -func cleanupPods(tCtx ktesting.TContext, namespace string, podNames []string, resourceType string) { - tCtx.Logf("Cleaning up %s resource pods after downgrade", resourceType) - for _, podName := range podNames { - tCtx.ExpectNoError(e2epod.DeletePodWithWaitByName(tCtx, tCtx.Client(), podName, namespace), "delete %s pod %s", resourceType, podName) - } - tCtx.Logf("Successfully cleaned up %s resource pods", resourceType) -} - -// createDeviceClassForExtendedResource creates a DeviceClass for testing extended resources. -// It sets up the builder state, creates the DeviceClass with a unique name, and returns -// the created DeviceClass and its corresponding resource name. -func createDeviceClassForExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, namespace string, useExplicit bool) (*resourceapi.DeviceClass, v1.ResourceName) { - var classIndex int - var nameSuffix string - - if useExplicit { - b.UseExtendedResourceName = true - classIndex = drautils.SingletonIndex - nameSuffix = "-extended-resource-explicit" - } else { - b.UseExtendedResourceName = false - classIndex = 1 - nameSuffix = "-extended-resource-implicit" - } - - // Create the DeviceClass with a unique name - deviceClass := b.Class(classIndex) - deviceClass.Name = namespace + nameSuffix - createdObjs := b.Create(tCtx, deviceClass) - deviceClass = createdObjs[0].(*resourceapi.DeviceClass) - - // Determine the resource name based on the type - var resourceName v1.ResourceName - if useExplicit { - // Verify ExtendedResourceName is present in the spec when using explicit resource name - // This ensures the API server hasn't stripped the field due to feature gates - require.NotNil(tCtx, deviceClass.Spec.ExtendedResourceName, "DeviceClass.Spec.ExtendedResourceName should be set for explicit resource name") - resourceName = v1.ResourceName(*deviceClass.Spec.ExtendedResourceName) - } else { - resourceName = v1.ResourceName(fmt.Sprintf("%s%s", resourceinternal.ResourceDeviceClassPrefix, deviceClass.Name)) - } - - tCtx.Logf("Created DeviceClass %q with extendedResourceName: %s", deviceClass.Name, resourceName) - - return deviceClass, resourceName -} - -// testExtendedResource tests a single extended resource type through the upgrade/downgrade cycle. -// It creates the DeviceClass, creates pods before upgrade, verifies them after upgrade, -// creates new pods after upgrade, and verifies and cleans up all pods after downgrade. -func testExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, useExplicit bool) upgradedTestFunc { - namespace := tCtx.Namespace() - - // Create DeviceClass and get the resource name - _, resourceName := createDeviceClassForExtendedResource(tCtx, b, namespace, useExplicit) - - // Determine resource type string for logging - resourceType := "implicit" - if useExplicit { - resourceType = "explicit" - } - - // Before upgrade: create and verify pod - podBefore := testExtendedResourcePod(tCtx, b, resourceName, fmt.Sprintf("%s-before-upgrade", resourceType)) - tCtx.Logf("Created pod %q for %s resource before upgrade", podBefore.Name, resourceType) - - return func(tCtx ktesting.TContext) downgradedTestFunc { - // After upgrade: verify existing pod still runs correctly - podBefore, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, podBefore.Name, metav1.GetOptions{}) - tCtx.ExpectNoError(err, "get %s pod after upgrade", resourceType) - require.Equal(tCtx, v1.PodRunning, podBefore.Status.Phase, "%s pod should still be running after upgrade", resourceType) - tCtx.Logf("Verified pod %q (%s resource) still running after upgrade", podBefore.Name, resourceType) - - // Create and verify another pod with the same extended resource - podAfter := testExtendedResourcePod(tCtx, b, resourceName, fmt.Sprintf("%s-after-upgrade", resourceType)) - tCtx.Logf("Created pod %q for %s resource after upgrade", podAfter.Name, resourceType) - - return func(tCtx ktesting.TContext) { - // After downgrade: verify all pods still run correctly - podNames := []string{podBefore.Name, podAfter.Name} - podUIDs := []string{string(podBefore.UID), string(podAfter.UID)} - verifyPodsRunning(tCtx, namespace, podNames, resourceType, "downgrade") - - // Clean up pods - cleanupPods(tCtx, namespace, podNames, resourceType) - - // Verify that special ResourceClaims for our pods are cleaned up by garbage collector - tCtx.Eventually(func(tCtx ktesting.TContext) int { - claims, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).List(tCtx, metav1.ListOptions{}) - if err != nil { - tCtx.Logf("Error listing claims: %v", err) - return -1 - } - specialClaimCount := 0 - for _, claim := range claims.Items { - for _, ownerRef := range claim.OwnerReferences { - // Only count claims owned by our specific pods - if ownerRef.Kind == "Pod" && slices.Contains(podUIDs, string(ownerRef.UID)) { - specialClaimCount++ - break - } - } - } - return specialClaimCount - }).WithTimeout(3*time.Minute).Should(gomega.Equal(0), "special ResourceClaims for %s pods should be garbage collected", resourceType) - - tCtx.Logf("Verified special ResourceClaims for %s resource were garbage collected", resourceType) - } - } -} - -// extendedResourceUpgradeDowngrade tests the DRAExtendedResources feature during upgrade/downgrade scenarios. -// This test verifies that: -// 1. DeviceClass with explicit extendedResourceName field (custom resource name) -// 2. DeviceClass with implicit extendedResourceName (using default format) -// 3. Pods requesting extended resources are scheduled and allocated devices -// 4. The scheduler creates special ResourceClaims for extended resource requests -// 5. Pod status contains extendedResourceClaimStatus mapping -// 6. Feature works correctly across upgrade/downgrade cycles -func extendedResourceUpgradeDowngrade(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { - // Test both explicit and implicit extended resources through upgrade/downgrade - explicitUpgradedFunc := testExtendedResource(tCtx, b, true) - implicitUpgradedFunc := testExtendedResource(tCtx, b, false) - - return func(tCtx ktesting.TContext) downgradedTestFunc { - explicitDowngradedFunc := explicitUpgradedFunc(tCtx) - implicitDowngradedFunc := implicitUpgradedFunc(tCtx) - - return func(tCtx ktesting.TContext) { - // Run downgrade verification and cleanup for both resource types - explicitDowngradedFunc(tCtx) - implicitDowngradedFunc(tCtx) - - tCtx.Logf("Extended resource upgrade/downgrade test completed successfully") - } - } + tCtx.Expect(foundMapping).To(gomega.BeTrueBecause("extendedResourceClaimStatus.RequestMappings should contain %s for pod %s", resourceName, pod.Name)) } diff --git a/test/e2e_dra/upgradedowngrade_test.go b/test/e2e_dra/upgradedowngrade_test.go index f8b0579aec0..76b78001cc4 100644 --- a/test/e2e_dra/upgradedowngrade_test.go +++ b/test/e2e_dra/upgradedowngrade_test.go @@ -76,7 +76,8 @@ var subTests = map[string]initialTestFunc{ "core DRA": coreDRA, "ResourceClaim device status": resourceClaimDeviceStatus, "DeviceTaints": deviceTaints, - "DRAExtendedResource": extendedResourceUpgradeDowngrade, + "ExplicitExtendedResource": extendedResourceUpgradeDowngrade(resourceTypeExplicit), + "ImplicitExtendedResource": extendedResourceUpgradeDowngrade(resourceTypeImplicit), } type initialTestFunc func(tCtx ktesting.TContext, builder *drautils.Builder) upgradedTestFunc