e2e: DRA: refactor device class creation API

Replace numeric index-based extended resource naming with descriptive
string-based naming in DRA E2E tests for improved readability and
maintainability.

Changes:
- Replaced ExtendedResourceName(int) with ExtendedResourceName(string)
  to use descriptive suffixes instead of numeric indices
- Replaced Class(int) with Class() and add ClassWithExtendedResource()
  for explicit extended resource creation
- Removed SingletonIndex constant in favor of e2enode.SampleDeviceResourceName
- Added DeviceClassWrapper with builder pattern methods
  (WithName, WithExtendedResource)
- Updated all test cases to use descriptive resource names ("cleanup",
  "quota", "overcommitment", etc.) instead of numeric indices (0, 1, 2)
- Used resourceapi.ResourceDeviceClassPrefix for implicit extended
  resource names

This refactoring  makes test code more readable and maintainable by making
extended resources and device class naming explicit rather than implicit through
index values.
This commit is contained in:
Ed Bartosh
2026-02-16 11:41:03 +02:00
parent 4edf082c40
commit 1b258b0de7
2 changed files with 115 additions and 89 deletions

View File

@@ -21,6 +21,7 @@ import (
_ "embed"
"errors"
"fmt"
"strconv"
"strings"
"sync"
"time"
@@ -53,6 +54,7 @@ import (
e2econformance "k8s.io/kubernetes/test/e2e/framework/conformance"
e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset"
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
dratest "k8s.io/kubernetes/test/integration/dra"
admissionapi "k8s.io/pod-security-admission/api"
@@ -2192,9 +2194,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
runExtendedClaimCleanup := func(ctx context.Context, b *drautils.Builder, shellScript string, waitFn func(context.Context, *v1.Pod) error, cleanupMessage string) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName("cleanup")
class := b.ClassWithExtendedResource(resourceName)
pod := b.Pod()
res := v1.ResourceList{}
res[v1.ResourceName(b.ExtendedResourceName(0))] = resource.MustParse("1")
res[v1.ResourceName(resourceName)] = resource.MustParse("1")
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
@@ -2203,7 +2207,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
pod.Spec.RestartPolicy = v1.RestartPolicyNever
ginkgo.By("Creating the pod with extended resource")
b.Create(tCtx, pod)
b.Create(tCtx, class, pod)
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
@@ -2237,15 +2241,16 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
nodes := drautils.NewNodes(f, 1, 1)
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false))
b := drautils.NewBuilder(f, driver)
b.UseExtendedResourceName = true
ginkgo.It("must run a pod with extended resource with resource quota", func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName("quota")
class := b.ClassWithExtendedResource(resourceName)
hard := v1.ResourceList{
v1.ResourceName("count/resourceclaims.resource.k8s.io"): resource.MustParse("10"),
v1.ResourceName(fmt.Sprintf("requests.%s", b.ExtendedResourceName(0))): resource.MustParse("10"),
v1.ResourceName(fmt.Sprintf("requests.%s", "deviceclass.resource.kubernetes.io/"+b.ClassName())): resource.MustParse("10"),
v1.ResourceName(fmt.Sprintf("%s.deviceclass.resource.k8s.io/devices", b.Class(0).Name)): resource.MustParse("10"),
v1.ResourceName("count/resourceclaims.resource.k8s.io"): resource.MustParse("10"),
v1.ResourceName(fmt.Sprintf("requests.%s", resourceName)): resource.MustParse("10"),
v1.ResourceName(fmt.Sprintf("requests.%s", resourceapi.ResourceDeviceClassPrefix+class.Name)): resource.MustParse("10"),
v1.ResourceName(fmt.Sprintf("%s.deviceclass.resource.k8s.io/devices", class.Name)): resource.MustParse("10"),
}
quota := &v1.ResourceQuota{
@@ -2256,13 +2261,16 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
}
b.Create(tCtx, quota)
// create a class with an extended resource
b.Create(tCtx, class)
pod := b.Pod()
res := v1.ResourceList{}
// b.ExtendedResourceName(0) is added to the device class with name: b.ClassName()+"0"
res[v1.ResourceName(b.ExtendedResourceName(0))] = resource.MustParse("1")
// explicit extended resource name
res[v1.ResourceName(resourceName)] = resource.MustParse("1")
// implicit extended resource name
res[v1.ResourceName("deviceclass.resource.kubernetes.io/"+b.ClassName())] = resource.MustParse("1")
res[v1.ResourceName(resourceapi.ResourceDeviceClassPrefix+class.Name)] = resource.MustParse("1")
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
@@ -2294,6 +2302,9 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
claim := b.ExternalClaim()
// Point the claim to the same class, so that it will consume the same extended
// resource and we can verify the total consumption in the resource quota.
claim.Spec.Devices.Requests[0].Exactly.DeviceClassName = class.Name
pod2 := b.PodExternal(claim.Name)
b.Create(tCtx, claim, pod2)
b.TestPod(tCtx, pod2)
@@ -2322,9 +2333,9 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
usedResources := v1.ResourceList{}
usedResources[v1.ResourceName("count/resourceclaims.resource.k8s.io")] = resource.MustParse("2")
usedResources[v1.ResourceName(fmt.Sprintf("requests.%s", b.ExtendedResourceName(0)))] = resource.MustParse(consumedDevices)
usedResources[v1.ResourceName(fmt.Sprintf("requests.%s", "deviceclass.resource.kubernetes.io/"+b.ClassName()))] = resource.MustParse(consumedDevices)
usedResources[v1.ResourceName(fmt.Sprintf("%s.deviceclass.resource.k8s.io/devices", b.Class(0).Name))] = resource.MustParse(consumedDevices)
usedResources[v1.ResourceName(fmt.Sprintf("requests.%s", resourceName))] = resource.MustParse(consumedDevices)
usedResources[v1.ResourceName(fmt.Sprintf("requests.%s", resourceapi.ResourceDeviceClassPrefix+class.Name))] = resource.MustParse(consumedDevices)
usedResources[v1.ResourceName(fmt.Sprintf("%s.deviceclass.resource.k8s.io/devices", class.Name))] = resource.MustParse(consumedDevices)
gomega.Eventually(ctx, framework.GetObject(f.ClientSet.CoreV1().ResourceQuotas(quota.Namespace).Get, quota.Name, metav1.GetOptions{})).
Should(gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
@@ -2337,11 +2348,16 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
})
ginkgo.It("must run a pod with both implicit and explicit extended resource with one container two resources", func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName("one-container-two-resources")
class := b.ClassWithExtendedResource(resourceName)
b.Create(tCtx, class)
extendedResourceTest(ctx, b, f, []string{
// implicit extended resource name
"deviceclass.resource.kubernetes.io/" + b.ClassName(),
// b.ExtendedResourceName(0) is added to the device class with name: b.ClassName()+"0"
b.ExtendedResourceName(0),
resourceapi.ResourceDeviceClassPrefix + class.Name,
// explicit extended resource name
resourceName,
}, []string{
"container_3_request_0", "true",
"container_3_request_1", "true",
@@ -2349,8 +2365,16 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
})
ginkgo.It("must run a pod with extended resource with one container one resource", func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName("one-container-one-resource")
class := b.ClassWithExtendedResource(resourceName)
b.Create(tCtx, class)
extendedResourceTest(ctx, b, f, []string{
b.ExtendedResourceName(0),
// implicit extended resource name
resourceapi.ResourceDeviceClassPrefix + class.Name,
// explicit extended resource name
resourceName,
}, []string{
"container_3_request_0", "true",
})
@@ -2358,17 +2382,16 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
ginkgo.It("must run a pod with extended resource with one container three resources", func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName("one-container-three-resources")
var objects []klog.KMetadata
for i := range 3 {
if i > 0 {
objects = append(objects, b.Class(i))
}
objects = append(objects, b.ClassWithExtendedResource(resourceName+strconv.Itoa(i)))
}
b.Create(tCtx, objects...)
extendedResourceTest(ctx, b, f, []string{
b.ExtendedResourceName(0),
b.ExtendedResourceName(1),
b.ExtendedResourceName(2),
resourceName + "0",
resourceName + "1",
resourceName + "2",
}, []string{
"container_3_request_0", "true",
"container_3_request_1", "true",
@@ -2386,14 +2409,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
pod.Spec.Containers[2].Name = "container2"
objects = append(objects, pod)
resourcePrefix := b.ExtendedResourceName("three-containers-one-resource-each")
for i := range 3 {
res := v1.ResourceList{}
res[v1.ResourceName(b.ExtendedResourceName(i))] = resource.MustParse("1")
resourceName := resourcePrefix + strconv.Itoa(i)
res[v1.ResourceName(resourceName)] = resource.MustParse("1")
pod.Spec.Containers[i].Resources.Requests = res
pod.Spec.Containers[i].Resources.Limits = res
if i > 0 {
objects = append(objects, b.Class(i))
}
objects = append(objects, b.ClassWithExtendedResource(resourceName))
}
b.Create(tCtx, objects...)
@@ -2416,23 +2439,25 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
pod.Spec.Containers[1].Name = "container1"
pod.Spec.Containers[2].Name = "container2"
resourcePrefix := b.ExtendedResourceName("three-containers-multiple-resources-each")
res := v1.ResourceList{}
res[v1.ResourceName(b.ExtendedResourceName(0))] = resource.MustParse("1")
res[v1.ResourceName(resourcePrefix+"0")] = resource.MustParse("1")
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
res = v1.ResourceList{}
res[v1.ResourceName(b.ExtendedResourceName(1))] = resource.MustParse("1")
res[v1.ResourceName(b.ExtendedResourceName(2))] = resource.MustParse("1")
res[v1.ResourceName(resourcePrefix+"1")] = resource.MustParse("1")
res[v1.ResourceName(resourcePrefix+"2")] = resource.MustParse("1")
pod.Spec.Containers[1].Resources.Requests = res
pod.Spec.Containers[1].Resources.Limits = res
res = v1.ResourceList{}
res[v1.ResourceName(b.ExtendedResourceName(3))] = resource.MustParse("1")
res[v1.ResourceName(b.ExtendedResourceName(4))] = resource.MustParse("1")
res[v1.ResourceName(b.ExtendedResourceName(5))] = resource.MustParse("1")
res[v1.ResourceName(resourcePrefix+"3")] = resource.MustParse("1")
res[v1.ResourceName(resourcePrefix+"4")] = resource.MustParse("1")
res[v1.ResourceName(resourcePrefix+"5")] = resource.MustParse("1")
pod.Spec.Containers[2].Resources.Requests = res
pod.Spec.Containers[2].Resources.Limits = res
for i := 1; i < 6; i++ {
objects = append(objects, b.Class(i))
for i := range 6 {
resourceName := resourcePrefix + strconv.Itoa(i)
objects = append(objects, b.ClassWithExtendedResource(resourceName))
}
objects = append(objects, pod)
@@ -2479,7 +2504,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
})
f.It("process extended resources after device plugin uninstall", f.WithSerial(), func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName(drautils.SingletonIndex)
resourceName := e2enode.SampleDeviceResourceName
extendedResourceName := deployDevicePlugin(tCtx, f, nodes.NodeNames[0:1], false)
gomega.Expect(string(extendedResourceName)).To(gomega.Equal(resourceName))
@@ -2495,7 +2520,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
b.Create(tCtx, b.Class(drautils.SingletonIndex), pod)
b.Create(tCtx, b.ClassWithExtendedResource(resourceName), pod)
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
@@ -2509,14 +2534,17 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
ginkgo.It("must prevent overcommitment of extended resources", func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName("overcommitment")
// Create a device class with extended resource
class := b.ClassWithExtendedResource(resourceName)
// Create first pod consuming all available resources
pod1 := b.Pod()
res := v1.ResourceList{
v1.ResourceName(b.ExtendedResourceName(0)): resource.MustParse("10"), // All available
v1.ResourceName(resourceName): resource.MustParse("10"), // All available
}
pod1.Spec.Containers[0].Resources.Requests = res
pod1.Spec.Containers[0].Resources.Limits = res
b.Create(tCtx, pod1)
b.Create(tCtx, class, pod1)
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod1)
framework.ExpectNoError(err, "start pod")
@@ -2557,8 +2585,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
ginkgo.It("must accurately populate ExtendedResourceClaimStatus", func(ctx context.Context) {
tCtx := f.TContext(ctx)
pod := b.Pod()
resource0 := b.ExtendedResourceName(0)
resource1 := b.ExtendedResourceName(1)
resource0 := b.ExtendedResourceName("claimstatus0")
resource1 := b.ExtendedResourceName("claimstatus1")
res := v1.ResourceList{
v1.ResourceName(resource0): resource.MustParse("2"),
v1.ResourceName(resource1): resource.MustParse("1"),
@@ -2566,7 +2594,9 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
b.Create(tCtx, b.Class(1), pod)
class0 := b.ClassWithExtendedResource(resource0)
class1 := b.ClassWithExtendedResource(resource1)
b.Create(tCtx, class0, class1, pod)
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
@@ -2598,9 +2628,13 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
f.It("must run a pod with extended resource requesting zero", func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName("zero")
class := b.ClassWithExtendedResource(resourceName)
b.Create(tCtx, class)
pod := b.Pod()
res := v1.ResourceList{
v1.ResourceName(b.ExtendedResourceName(0)): resource.MustParse("0"),
v1.ResourceName(resourceName): resource.MustParse("0"),
}
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
@@ -2617,7 +2651,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
f.It("must process extended resource after Device Class creation", func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName(1)
resourceName := b.ExtendedResourceName("after-device-class-creation")
res := v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")}
pod := b.Pod()
pod.Spec.Containers[0].Resources.Requests = res
@@ -2629,7 +2663,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
framework.ExpectNoError(err, "pod should be unschedulable before device class creation")
ginkgo.By("Creating Device Class after pod creation")
b.Create(tCtx, b.Class(1))
b.Create(tCtx, b.ClassWithExtendedResource(resourceName))
err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
@@ -2655,17 +2689,15 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
tCtx := f.TContext(ctx)
var objects []klog.KMetadata
extendedResourceName := deployDevicePlugin(tCtx, f, nodes.ExtraNodeNames, false)
// b.ExtendedResourceName(SingletonIndex) must be the same as the returned extendedResourceName.
// b.ExtendedResourceName(SingletonIndex) is used for DRA drivers whereas
// extendedResourceName is used for device plugin.
gomega.Expect(string(extendedResourceName)).To(gomega.Equal(b.ExtendedResourceName(drautils.SingletonIndex)))
// extendedResourceName should be the same as the one used by Device Plugin
gomega.Expect(string(extendedResourceName)).To(gomega.Equal(e2enode.SampleDeviceResourceName))
pod1 := b.Pod()
res := v1.ResourceList{}
res[v1.ResourceName(b.ExtendedResourceName(drautils.SingletonIndex))] = resource.MustParse("2")
res[v1.ResourceName(e2enode.SampleDeviceResourceName)] = resource.MustParse("2")
pod1.Spec.Containers[0].Resources.Requests = res
pod1.Spec.Containers[0].Resources.Limits = res
objects = append(objects, b.Class(drautils.SingletonIndex), pod1)
objects = append(objects, b.ClassWithExtendedResource(e2enode.SampleDeviceResourceName), pod1)
pod2 := b.Pod()
pod2.Spec.Containers[0].Resources.Requests = res

View File

@@ -22,7 +22,6 @@ import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"time"
@@ -44,24 +43,16 @@ import (
"k8s.io/klog/v2"
"k8s.io/kubernetes/test/e2e/dra/test-driver/app"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
"k8s.io/kubernetes/test/utils/ktesting"
admissionapi "k8s.io/pod-security-admission/api"
"k8s.io/utils/ptr"
)
// ExtendedResourceName returns hard coded extended resource name with a variable
// suffix from the input integer when it's greater than or equal to 0.
// i == -1 == SingletonIndex is special, the extended resource name has no extra suffix
// and matches the one used by the example device plugin.
func (b *Builder) ExtendedResourceName(i int) string {
switch i {
case SingletonIndex:
return e2enode.SampleDeviceResourceName
default:
return b.Driver.Name + "/resource" + fmt.Sprintf("-%d", i)
}
// ExtendedResourceName returns extended resource name with a variable suffix.
// Example: b.ExtendedResourceName("gpu") returns "driver-name/resource-gpu"
func (b *Builder) ExtendedResourceName(suffix string) string {
return b.Driver.Name + "/resource-" + suffix
}
// Builder contains a running counter to make objects unique within their
@@ -77,42 +68,26 @@ type Builder struct {
SkipCleanup bool
}
// DeviceClassWrapper is a wrapper around DeviceClass that allows
// adding builder-style functions that modify the class before creation.
type DeviceClassWrapper struct {
*resourceapi.DeviceClass
}
// ClassName returns the default device class name.
func (b *Builder) ClassName() string {
return b.namespace + b.Driver.NameSuffix + "-class"
}
// SingletonIndex causes Builder.Class and ExtendedResourceName to create a
// DeviceClass where the the extended resource name has no %d
// suffix and matches the name as used by the example device plugin.
const SingletonIndex = -1
// Class returns the device Class that the builder's other objects
// reference.
// The input i is used to pick the extended resource name whose suffix has the
// same i for the device class.
// i == -1 == SingletonIndex is special, the extended resource name has no extra suffix.
func (b *Builder) Class(i int) *resourceapi.DeviceClass {
ern := b.ExtendedResourceName(i)
func (b *Builder) Class() *DeviceClassWrapper {
name := b.ClassName()
switch i {
case SingletonIndex:
name += "-singleton"
case 0:
// No numeric suffix. This is what most tests use.
default:
name += "-" + strconv.Itoa(i)
}
class := &resourceapi.DeviceClass{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
}
if b.UseExtendedResourceName {
class.Spec = resourceapi.DeviceClassSpec{
ExtendedResourceName: &ern,
}
}
class.Spec.Selectors = []resourceapi.DeviceSelector{{
CEL: &resourceapi.CELDeviceSelector{
Expression: fmt.Sprintf(`device.driver == "%s"`, b.Driver.Name),
@@ -128,7 +103,26 @@ func (b *Builder) Class(i int) *resourceapi.DeviceClass {
},
}}
}
return class
return &DeviceClassWrapper{DeviceClass: class}
}
// ClassWithExtendedResource returns a device class with the extended resource name set to the provided value.
// The class name is suffixed with the last part of the extended resource name to make it unique.
func (b *Builder) ClassWithExtendedResource(extendedResource string) *resourceapi.DeviceClass {
suffix := extendedResource[strings.LastIndex(extendedResource, "/")+1:]
return b.Class().WithName(b.ClassName() + "-" + suffix).WithExtendedResource(extendedResource).DeviceClass
}
// WithName sets the name of the device class.
func (dcw *DeviceClassWrapper) WithName(name string) *DeviceClassWrapper {
dcw.ObjectMeta.Name = name
return dcw
}
// WithExtendedResource sets the extended resource name of the device class.
func (dcw *DeviceClassWrapper) WithExtendedResource(extendedResourceName string) *DeviceClassWrapper {
dcw.Spec.ExtendedResourceName = &extendedResourceName
return dcw
}
// ExternalClaim returns external resource claim
@@ -490,7 +484,7 @@ func (b *Builder) setUp(tCtx ktesting.TContext) {
b.namespace = tCtx.Namespace()
b.podCounter = 0
b.claimCounter = 0
b.Create(tCtx, b.Class(0))
b.Create(tCtx, b.Class().DeviceClass)
tCtx.CleanupCtx(b.tearDown)
}