mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-18 04:54:54 +00:00
Merge pull request #136568 from sairameshv/extRes_test
KEP-5004: Add upgrade/downgrade e2e tests for the DRAExtendedResources feature
This commit is contained in:
Kubernetes Prow Robot
GitHub
198
test/e2e_dra/extendedresources_test.go
Normal file
198
test/e2e_dra/extendedresources_test.go
Normal file
@@ -0,0 +1,198 @@
|
||||
/*
|
||||
Copyright The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2edra
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/gomega"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourceapi "k8s.io/api/resource/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
drautils "k8s.io/kubernetes/test/e2e/dra/utils"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
"k8s.io/kubernetes/test/utils/ktesting"
|
||||
)
|
||||
|
||||
const (
|
||||
resourceTypeExplicit = "explicit"
|
||||
resourceTypeImplicit = "implicit"
|
||||
)
|
||||
|
||||
// extendedResourceUpgradeDowngrade tests the DRAExtendedResources feature during upgrade/downgrade scenarios
|
||||
// for a specific resource type (explicit or implicit).
|
||||
// This test verifies:
|
||||
// 1. DeviceClasses can be created with the specified extendedResourceName type
|
||||
// 2. Pods requesting extended resources are scheduled and allocated devices before upgrade
|
||||
// 3. Existing pods continue running correctly after upgrade
|
||||
// 4. New pods with extended resources work correctly after upgrade
|
||||
// 5. All pods continue running correctly after downgrade
|
||||
// 6. ResourceClaims are garbage collected when pods are deleted
|
||||
func extendedResourceUpgradeDowngrade(resourceType string) initialTestFunc {
|
||||
return func(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc {
|
||||
return testExtendedResource(tCtx, b, resourceType)
|
||||
}
|
||||
}
|
||||
|
||||
// testExtendedResource tests a single extended resource type through the upgrade/downgrade cycle.
|
||||
// It creates the DeviceClass, creates pods before upgrade, verifies them after upgrade,
|
||||
// creates new pods after upgrade, and verifies and cleans up all pods after downgrade.
|
||||
func testExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, resourceType string) upgradedTestFunc {
|
||||
namespace := tCtx.Namespace()
|
||||
|
||||
// Create DeviceClass and get the resource name
|
||||
resourceName := createDeviceClassForExtendedResource(tCtx, b, resourceType)
|
||||
|
||||
// Before upgrade: create and verify pod
|
||||
podBefore := createPodWithExtendedResource(tCtx, b, fmt.Sprintf("%s-before-upgrade", resourceType), resourceName)
|
||||
verifyPodRunningWithClaim(tCtx, b, podBefore, resourceName)
|
||||
|
||||
return func(tCtx ktesting.TContext) downgradedTestFunc {
|
||||
// After upgrade: verify existing pod still runs correctly
|
||||
verifyPodRunningWithClaim(tCtx, b, podBefore, resourceName)
|
||||
|
||||
// Create and verify another pod with the same extended resource
|
||||
podAfter := createPodWithExtendedResource(tCtx, b, fmt.Sprintf("%s-after-upgrade", resourceType), resourceName)
|
||||
verifyPodRunningWithClaim(tCtx, b, podAfter, resourceName)
|
||||
|
||||
return func(tCtx ktesting.TContext) {
|
||||
// After downgrade: verify all pods still run correctly with their claims
|
||||
podNames := []string{podBefore.Name, podAfter.Name}
|
||||
claimNames := make([]string, 0, len(podNames))
|
||||
|
||||
for _, podName := range podNames {
|
||||
pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, podName, metav1.GetOptions{})
|
||||
tCtx.ExpectNoError(err, "get %s pod %s", resourceType, podName)
|
||||
verifyPodRunningWithClaim(tCtx, b, pod, resourceName)
|
||||
|
||||
// Collect claim name before cleanup
|
||||
if pod.Status.ExtendedResourceClaimStatus != nil {
|
||||
claimNames = append(claimNames, pod.Status.ExtendedResourceClaimStatus.ResourceClaimName)
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up pods
|
||||
tCtx.Logf("Cleaning up %s resource pods after downgrade", resourceType)
|
||||
for _, podName := range podNames {
|
||||
tCtx.ExpectNoError(e2epod.DeletePodWithWaitByName(tCtx, tCtx.Client(), podName, namespace), "delete %s pod %s", resourceType, podName)
|
||||
}
|
||||
tCtx.Logf("Successfully cleaned up %s resource pods", resourceType)
|
||||
|
||||
// Verify that ResourceClaims for our pods are cleaned up by garbage collector
|
||||
remainingClaims := claimNames
|
||||
tCtx.Eventually(func(tCtx ktesting.TContext) int {
|
||||
stillExisting := make([]string, 0, len(remainingClaims))
|
||||
for _, claimName := range remainingClaims {
|
||||
_, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claimName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
stillExisting = append(stillExisting, claimName)
|
||||
} else if !apierrors.IsNotFound(err) {
|
||||
// If we get a real API error (not NotFound), fail immediately
|
||||
tCtx.ExpectNoError(err, "unexpected error checking ResourceClaim %s", claimName)
|
||||
}
|
||||
// If IsNotFound, the claim has been deleted (expected behavior)
|
||||
}
|
||||
remainingClaims = stillExisting
|
||||
return len(remainingClaims)
|
||||
}).WithTimeout(3*time.Minute).Should(gomega.Equal(0), "ResourceClaims for %s pods should be garbage collected", resourceType)
|
||||
|
||||
tCtx.Logf("Verified ResourceClaims for %s resource were garbage collected", resourceType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// createDeviceClassForExtendedResource creates a DeviceClass for testing extended resources.
|
||||
// It creates the DeviceClass with a unique name, and returns its corresponding resource name.
|
||||
func createDeviceClassForExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, resourceType string) v1.ResourceName {
|
||||
var deviceClass *resourceapi.DeviceClass
|
||||
var resourceName v1.ResourceName
|
||||
|
||||
if resourceType == resourceTypeExplicit {
|
||||
// Framework automatically names it: b.ClassName() + "-explicit"
|
||||
resourceName = v1.ResourceName(b.ExtendedResourceName(resourceTypeExplicit))
|
||||
deviceClass = b.ClassWithExtendedResource(string(resourceName))
|
||||
} else {
|
||||
// Framework automatically names it
|
||||
deviceClass = b.Class().DeviceClass
|
||||
resourceName = v1.ResourceName(fmt.Sprintf("%s%s", resourceapi.ResourceDeviceClassPrefix, deviceClass.Name))
|
||||
}
|
||||
|
||||
createdObjs := b.Create(tCtx, deviceClass)
|
||||
|
||||
// Verify ExtendedResourceName is present in the spec when using explicit resource name
|
||||
if resourceType == resourceTypeExplicit {
|
||||
deviceClass = createdObjs[0].(*resourceapi.DeviceClass)
|
||||
tCtx.Expect(deviceClass.Spec.ExtendedResourceName).ToNot(gomega.BeNil(), "DeviceClass.Spec.ExtendedResourceName should be set for explicit resource name")
|
||||
}
|
||||
|
||||
tCtx.Logf("Created DeviceClass %q with extendedResourceName: %s", deviceClass.Name, resourceName)
|
||||
|
||||
return resourceName
|
||||
}
|
||||
|
||||
// createPodWithExtendedResource creates a pod with the given name requesting an extended resource.
|
||||
func createPodWithExtendedResource(tCtx ktesting.TContext, b *drautils.Builder, podName string, resourceName v1.ResourceName) *v1.Pod {
|
||||
pod := b.Pod()
|
||||
pod.Name = podName
|
||||
resources := v1.ResourceList{resourceName: resource.MustParse("1")}
|
||||
pod.Spec.Containers[0].Resources.Requests = resources
|
||||
pod.Spec.Containers[0].Resources.Limits = resources
|
||||
pod = b.Create(tCtx, pod)[0].(*v1.Pod)
|
||||
tCtx.Logf("Created pod %q requesting extended resource %q", pod.Name, resourceName)
|
||||
return pod
|
||||
}
|
||||
|
||||
// verifyPodRunningWithClaim verifies a pod is running and has a valid ResourceClaim.
|
||||
func verifyPodRunningWithClaim(tCtx ktesting.TContext, b *drautils.Builder, pod *v1.Pod, resourceName v1.ResourceName) {
|
||||
namespace := tCtx.Namespace()
|
||||
b.TestPod(tCtx, pod)
|
||||
tCtx.Logf("Pod %q is running", pod.Name)
|
||||
|
||||
// Get updated pod to check status
|
||||
pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{})
|
||||
tCtx.ExpectNoError(err, "get pod to check status")
|
||||
|
||||
// Verify pod.status.extendedResourceClaimStatus contains the mapping
|
||||
tCtx.Expect(pod.Status.ExtendedResourceClaimStatus).ToNot(gomega.BeNil(), "pod.status.extendedResourceClaimStatus should not be nil for pod %s", pod.Name)
|
||||
|
||||
// Get the claim name from pod status
|
||||
claimName := pod.Status.ExtendedResourceClaimStatus.ResourceClaimName
|
||||
tCtx.Logf("Verified extendedResourceClaimStatus.ResourceClaimName: %s for pod %q", claimName, pod.Name)
|
||||
|
||||
// Fetch the ResourceClaim using the name from pod status
|
||||
claim, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Get(tCtx, claimName, metav1.GetOptions{})
|
||||
tCtx.ExpectNoError(err, "getting ResourceClaim %s", claimName)
|
||||
|
||||
// Verify the claim is allocated
|
||||
tCtx.Expect(claim.Status.Allocation).ToNot(gomega.BeNil(), "ResourceClaim %s should be allocated", claim.Name)
|
||||
tCtx.Logf("ResourceClaim %q is allocated for pod %q", claim.Name, pod.Name)
|
||||
|
||||
// Verify request mappings contain the extended resource
|
||||
foundMapping := slices.ContainsFunc(pod.Status.ExtendedResourceClaimStatus.RequestMappings, func(reqMapping v1.ContainerExtendedResourceRequest) bool {
|
||||
if reqMapping.ResourceName == string(resourceName) {
|
||||
tCtx.Logf("Found request mapping for pod %q: container=%s, resource=%s, request=%s",
|
||||
pod.Name, reqMapping.ContainerName, reqMapping.ResourceName, reqMapping.RequestName)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
tCtx.Expect(foundMapping).To(gomega.BeTrueBecause("extendedResourceClaimStatus.RequestMappings should contain %s for pod %s", resourceName, pod.Name))
|
||||
}
|
||||
@@ -76,6 +76,8 @@ var subTests = map[string]initialTestFunc{
|
||||
"core DRA": coreDRA,
|
||||
"ResourceClaim device status": resourceClaimDeviceStatus,
|
||||
"DeviceTaints": deviceTaints,
|
||||
"ExplicitExtendedResource": extendedResourceUpgradeDowngrade(resourceTypeExplicit),
|
||||
"ImplicitExtendedResource": extendedResourceUpgradeDowngrade(resourceTypeImplicit),
|
||||
}
|
||||
|
||||
type initialTestFunc func(tCtx ktesting.TContext, builder *drautils.Builder) upgradedTestFunc
|
||||
@@ -213,7 +215,7 @@ func testUpgradeDowngrade(tCtx ktesting.TContext) {
|
||||
cluster = localupcluster.New(tCtx)
|
||||
localUpClusterEnv := map[string]string{
|
||||
"RUNTIME_CONFIG": "resource.k8s.io/v1beta1,resource.k8s.io/v1beta2,resource.k8s.io/v1alpha3",
|
||||
"FEATURE_GATES": "DynamicResourceAllocation=true,DRADeviceTaintRules=true,DRADeviceTaints=true",
|
||||
"FEATURE_GATES": "DynamicResourceAllocation=true,DRADeviceTaintRules=true,DRADeviceTaints=true,DRAExtendedResource=true",
|
||||
// *not* needed because driver will run in "local filesystem" mode (= driver.IsLocal): "ALLOW_PRIVILEGED": "1",
|
||||
}
|
||||
cluster.Start(tCtx, binDir, localUpClusterEnv)
|
||||
|
||||
Reference in New Issue
Block a user