From 7555cbca90f941a6b71a8ec060c756c876a0c927 Mon Sep 17 00:00:00 2001 From: Morten Torkildsen Date: Fri, 28 Feb 2025 19:33:20 +0000 Subject: [PATCH] DRA: Updates the e2e tests for Prioritized Alternatives in Device Requests --- .../resourceclaim/resourceclaim.go | 35 ++ .../resourceclaim/resourceclaim_test.go | 244 +++++++++++ test/e2e/dra/dra.go | 409 ++++++++++++++++++ test/e2e/dra/test-driver/app/kubeletplugin.go | 28 +- test/e2e/feature/feature.go | 13 + 5 files changed, 718 insertions(+), 11 deletions(-) diff --git a/staging/src/k8s.io/dynamic-resource-allocation/resourceclaim/resourceclaim.go b/staging/src/k8s.io/dynamic-resource-allocation/resourceclaim/resourceclaim.go index e2c895f3499..d25c70f5e1a 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/resourceclaim/resourceclaim.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/resourceclaim/resourceclaim.go @@ -26,6 +26,8 @@ package resourceclaim import ( "errors" "fmt" + "slices" + "strings" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1beta1" @@ -106,3 +108,36 @@ func CanBeReserved(claim *resourceapi.ResourceClaim) bool { // Currently no restrictions on sharing... return true } + +// BaseRequestRef returns the request name if the reference is to a top-level +// request and the name of the parent request if the reference is to a subrequest. +func BaseRequestRef(requestRef string) string { + segments := strings.Split(requestRef, "/") + return segments[0] +} + +// ConfigForResult returns the configs that are applicable to device +// allocated for the provided result. +func ConfigForResult(deviceConfigurations []resourceapi.DeviceAllocationConfiguration, result resourceapi.DeviceRequestAllocationResult) []resourceapi.DeviceAllocationConfiguration { + var configs []resourceapi.DeviceAllocationConfiguration + for _, deviceConfiguration := range deviceConfigurations { + if deviceConfiguration.Opaque != nil && + isMatch(deviceConfiguration.Requests, result.Request) { + configs = append(configs, deviceConfiguration) + } + } + return configs +} + +func isMatch(requests []string, requestRef string) bool { + if len(requests) == 0 { + return true + } + + if slices.Contains(requests, requestRef) { + return true + } + + baseRequestRef := BaseRequestRef(requestRef) + return slices.Contains(requests, baseRequestRef) +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/resourceclaim/resourceclaim_test.go b/staging/src/k8s.io/dynamic-resource-allocation/resourceclaim/resourceclaim_test.go index 9f9804c7fce..1adcfc8f723 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/resourceclaim/resourceclaim_test.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/resourceclaim/resourceclaim_test.go @@ -332,3 +332,247 @@ func TestName(t *testing.T) { }) } } + +func TestBaseRequestRef(t *testing.T) { + testcases := map[string]struct { + requestRef string + expectedBaseRequestRef string + }{ + "valid-no-subrequest": { + requestRef: "foo", + expectedBaseRequestRef: "foo", + }, + "valid-subrequest": { + requestRef: "foo/bar", + expectedBaseRequestRef: "foo", + }, + } + + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + baseRequestRef := BaseRequestRef(tc.requestRef) + assert.Equal(t, tc.expectedBaseRequestRef, baseRequestRef) + }) + } +} + +func TestConfigForResult(t *testing.T) { + testcases := map[string]struct { + deviceConfigurations []resourceapi.DeviceAllocationConfiguration + result resourceapi.DeviceRequestAllocationResult + expectedConfigs []resourceapi.DeviceAllocationConfiguration + }{ + "opaque-nil": { + deviceConfigurations: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: nil, + }, + }, + }, + result: resourceapi.DeviceRequestAllocationResult{ + Request: "foo", + Device: "device-1", + }, + expectedConfigs: nil, + }, + "empty-requests-match-all": { + deviceConfigurations: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + result: resourceapi.DeviceRequestAllocationResult{ + Request: "foo", + Device: "device-1", + }, + expectedConfigs: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + }, + "match-regular-request": { + deviceConfigurations: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + result: resourceapi.DeviceRequestAllocationResult{ + Request: "foo", + Device: "device-1", + }, + expectedConfigs: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + }, + "match-parent-request-for-subrequest": { + deviceConfigurations: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + result: resourceapi.DeviceRequestAllocationResult{ + Request: "foo/bar", + Device: "device-1", + }, + expectedConfigs: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + }, + "match-subrequest": { + deviceConfigurations: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo/bar", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo/not-bar", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + result: resourceapi.DeviceRequestAllocationResult{ + Request: "foo/bar", + Device: "device-1", + }, + expectedConfigs: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo/bar", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + }, + "match-both-source-class-and-claim": { + deviceConfigurations: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + { + Source: resourceapi.AllocationConfigSourceClaim, + Requests: []string{ + "foo/bar", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + result: resourceapi.DeviceRequestAllocationResult{ + Request: "foo/bar", + Device: "device-1", + }, + expectedConfigs: []resourceapi.DeviceAllocationConfiguration{ + { + Source: resourceapi.AllocationConfigSourceClass, + Requests: []string{ + "foo", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + { + Source: resourceapi.AllocationConfigSourceClaim, + Requests: []string{ + "foo/bar", + }, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: "driver-a", + }, + }, + }, + }, + }, + } + + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + configs := ConfigForResult(tc.deviceConfigurations, tc.result) + assert.Equal(t, tc.expectedConfigs, configs) + }) + } +} diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index 4b8250f2fb7..e03479ecaa2 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -863,6 +863,411 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) } + prioritizedListTests := func() { + nodes := NewNodes(f, 1, 1) + + driver1Params, driver1Env := `{"driver":"1"}`, []string{"admin_driver", "1"} + driver2Params, driver2Env := `{"driver":"2"}`, []string{"admin_driver", "2"} + + driver1 := NewDriver(f, nodes, perNode(-1, nodes), []map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + { + "device-1-1": { + "dra.example.com/version": {StringValue: ptr.To("1.0.0")}, + "dra.example.com/pcieRoot": {StringValue: ptr.To("bar")}, + }, + "device-1-2": { + "dra.example.com/version": {StringValue: ptr.To("2.0.0")}, + "dra.example.com/pcieRoot": {StringValue: ptr.To("foo")}, + }, + }, + }...) + driver1.NameSuffix = "-1" + b1 := newBuilder(f, driver1) + b1.classParameters = driver1Params + + driver2 := NewDriver(f, nodes, perNode(-1, nodes), []map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + { + "device-2-1": { + "dra.example.com/version": {StringValue: ptr.To("1.0.0")}, + "dra.example.com/pcieRoot": {StringValue: ptr.To("foo")}, + }, + }, + }...) + driver2.NameSuffix = "-2" + b2 := newBuilder(f, driver2) + b2.classParameters = driver2Params + + f.It("selects the first subrequest that can be satisfied", feature.DRAPrioritizedList, func(ctx context.Context) { + name := "external-multiclaim" + params := `{"a":"b"}` + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{{ + Name: "request-1", + FirstAvailable: []resourceapi.DeviceSubRequest{ + { + Name: "sub-request-1", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 3, + }, + { + Name: "sub-request-2", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 2, + }, + { + Name: "sub-request-3", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + }}, + Config: []resourceapi.DeviceClaimConfiguration{ + { + Requests: []string{"request-1"}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b1.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(params), + }, + }, + }, + }, + }, + }, + }, + } + pod := b1.podExternal() + podClaimName := "resource-claim" + externalClaimName := "external-multiclaim" + pod.Spec.ResourceClaims = []v1.PodResourceClaim{ + { + Name: podClaimName, + ResourceClaimName: &externalClaimName, + }, + } + b1.create(ctx, claim, pod) + b1.testPod(ctx, f, pod) + + var allocatedResourceClaim *resourceapi.ResourceClaim + gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) { + var err error + allocatedResourceClaim, err = f.ClientSet.ResourceV1beta1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) + return allocatedResourceClaim, err + }).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil))) + results := allocatedResourceClaim.Status.Allocation.Devices.Results + gomega.Expect(results).To(gomega.HaveLen(2)) + gomega.Expect(results[0].Request).To(gomega.Equal("request-1/sub-request-2")) + gomega.Expect(results[1].Request).To(gomega.Equal("request-1/sub-request-2")) + }) + + f.It("uses the config for the selected subrequest", feature.DRAPrioritizedList, func(ctx context.Context) { + name := "external-multiclaim" + parentReqParams, parentReqEnv := `{"a":"b"}`, []string{"user_a", "b"} + subReq1Params := `{"c":"d"}` + subReq2Params, subReq2Env := `{"e":"f"}`, []string{"user_e", "f"} + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{{ + Name: "request-1", + FirstAvailable: []resourceapi.DeviceSubRequest{ + { + Name: "sub-request-1", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 3, + }, + { + Name: "sub-request-2", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 2, + }, + }, + }}, + Config: []resourceapi.DeviceClaimConfiguration{ + { + Requests: []string{"request-1"}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b1.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(parentReqParams), + }, + }, + }, + }, + { + Requests: []string{"request-1/sub-request-1"}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b1.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(subReq1Params), + }, + }, + }, + }, + { + Requests: []string{"request-1/sub-request-2"}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b1.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(subReq2Params), + }, + }, + }, + }, + }, + }, + }, + } + pod := b1.podExternal() + podClaimName := "resource-claim" + externalClaimName := "external-multiclaim" + pod.Spec.ResourceClaims = []v1.PodResourceClaim{ + { + Name: podClaimName, + ResourceClaimName: &externalClaimName, + }, + } + b1.create(ctx, claim, pod) + var expectedEnv []string + expectedEnv = append(expectedEnv, parentReqEnv...) + expectedEnv = append(expectedEnv, subReq2Env...) + b1.testPod(ctx, f, pod, expectedEnv...) + }) + + f.It("chooses the correct subrequest subject to constraints", feature.DRAPrioritizedList, func(ctx context.Context) { + name := "external-multiclaim" + params := `{"a":"b"}` + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{ + { + Name: "request-1", + FirstAvailable: []resourceapi.DeviceSubRequest{ + { + Name: "sub-request-1", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 1, + }, + { + Name: "sub-request-2", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + }, + { + Name: "request-2", + DeviceClassName: b2.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + Constraints: []resourceapi.DeviceConstraint{ + { + Requests: []string{"request-1", "request-2"}, + MatchAttribute: ptr.To(resourceapi.FullyQualifiedName("dra.example.com/version")), + }, + { + Requests: []string{"request-1/sub-request-1", "request-2"}, + MatchAttribute: ptr.To(resourceapi.FullyQualifiedName("dra.example.com/pcieRoot")), + }, + }, + Config: []resourceapi.DeviceClaimConfiguration{ + { + Requests: []string{}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b1.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(params), + }, + }, + }, + }, + }, + }, + }, + } + pod := b1.podExternal() + podClaimName := "resource-claim" + externalClaimName := "external-multiclaim" + pod.Spec.ResourceClaims = []v1.PodResourceClaim{ + { + Name: podClaimName, + ResourceClaimName: &externalClaimName, + }, + } + b1.create(ctx, claim, pod) + b1.testPod(ctx, f, pod) + + var allocatedResourceClaim *resourceapi.ResourceClaim + gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) { + var err error + allocatedResourceClaim, err = f.ClientSet.ResourceV1beta1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) + return allocatedResourceClaim, err + }).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil))) + results := allocatedResourceClaim.Status.Allocation.Devices.Results + gomega.Expect(results).To(gomega.HaveLen(2)) + gomega.Expect(results[0].Request).To(gomega.Equal("request-1/sub-request-2")) + gomega.Expect(results[1].Request).To(gomega.Equal("request-2")) + }) + + f.It("filters config correctly for multiple devices", feature.DRAPrioritizedList, func(ctx context.Context) { + name := "external-multiclaim" + req1Params, req1Env := `{"a":"b"}`, []string{"user_a", "b"} + req1subReq1Params, _ := `{"c":"d"}`, []string{"user_d", "d"} + req1subReq2Params, req1subReq2Env := `{"e":"f"}`, []string{"user_e", "f"} + req2Params, req2Env := `{"g":"h"}`, []string{"user_g", "h"} + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{ + { + Name: "request-1", + FirstAvailable: []resourceapi.DeviceSubRequest{ + { + Name: "sub-request-1", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 20, // Requests more than are available. + }, + { + Name: "sub-request-2", + DeviceClassName: b1.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + }, + { + Name: "request-2", + DeviceClassName: b2.className(), + AllocationMode: resourceapi.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + Config: []resourceapi.DeviceClaimConfiguration{ + { + Requests: []string{"request-1"}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b1.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(req1Params), + }, + }, + }, + }, + { + Requests: []string{"request-1/sub-request-1"}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b1.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(req1subReq1Params), + }, + }, + }, + }, + { + Requests: []string{"request-1/sub-request-2"}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b1.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(req1subReq2Params), + }, + }, + }, + }, + { + Requests: []string{"request-2"}, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b2.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(req2Params), + }, + }, + }, + }, + }, + }, + }, + } + pod := b1.pod() + pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy()) + pod.Spec.Containers[0].Name = "with-resource-0" + pod.Spec.Containers[1].Name = "with-resource-1" + pod.Spec.ResourceClaims = []v1.PodResourceClaim{ + { + Name: name, + ResourceClaimName: &name, + }, + } + pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: name, Request: "request-1"}} + pod.Spec.Containers[1].Resources.Claims = []v1.ResourceClaim{{Name: name, Request: "request-2"}} + + b1.create(ctx, claim, pod) + err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) + framework.ExpectNoError(err, "start pod") + + var allocatedResourceClaim *resourceapi.ResourceClaim + gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) { + var err error + allocatedResourceClaim, err = f.ClientSet.ResourceV1beta1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) + return allocatedResourceClaim, err + }).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil))) + results := allocatedResourceClaim.Status.Allocation.Devices.Results + gomega.Expect(results).To(gomega.HaveLen(2)) + gomega.Expect(results[0].Request).To(gomega.Equal("request-1/sub-request-2")) + gomega.Expect(results[1].Request).To(gomega.Equal("request-2")) + + req1ExpectedEnv := []string{ + "claim_external_multiclaim_request_1", + "true", + } + req1ExpectedEnv = append(req1ExpectedEnv, req1Env...) + req1ExpectedEnv = append(req1ExpectedEnv, req1subReq2Env...) + req1ExpectedEnv = append(req1ExpectedEnv, driver1Env...) + testContainerEnv(ctx, f, pod, "with-resource-0", true, req1ExpectedEnv...) + + req2ExpectedEnv := []string{ + "claim_external_multiclaim_request_2", + "true", + } + req2ExpectedEnv = append(req2ExpectedEnv, req2Env...) + req2ExpectedEnv = append(req2ExpectedEnv, driver2Env...) + testContainerEnv(ctx, f, pod, "with-resource-1", true, req2ExpectedEnv...) + }) + } + ginkgo.Context("on single node", func() { singleNodeTests() }) @@ -871,6 +1276,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, multiNodeTests() }) + ginkgo.Context("with prioritized list", func() { + prioritizedListTests() + }) + // TODO (https://github.com/kubernetes/kubernetes/issues/123699): move most of the test below into `testDriver` so that they get // executed with different parameters. diff --git a/test/e2e/dra/test-driver/app/kubeletplugin.go b/test/e2e/dra/test-driver/app/kubeletplugin.go index 9fec1b9c913..863477c54e7 100644 --- a/test/e2e/dra/test-driver/app/kubeletplugin.go +++ b/test/e2e/dra/test-driver/app/kubeletplugin.go @@ -24,7 +24,6 @@ import ( "os" "path/filepath" "regexp" - "slices" "sort" "strings" "sync" @@ -38,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceclaim" "k8s.io/dynamic-resource-allocation/resourceslice" "k8s.io/klog/v2" drapbv1alpha4 "k8s.io/kubelet/pkg/apis/dra/v1alpha4" @@ -103,7 +103,8 @@ var _ drapb.DRAPluginServer = &ExamplePlugin{} // getJSONFilePath returns the absolute path where CDI file is/should be. func (ex *ExamplePlugin) getJSONFilePath(claimUID string, requestName string) string { - return filepath.Join(ex.cdiDir, fmt.Sprintf("%s-%s-%s.json", ex.driverName, claimUID, requestName)) + baseRequestRef := resourceclaim.BaseRequestRef(requestName) + return filepath.Join(ex.cdiDir, fmt.Sprintf("%s-%s-%s.json", ex.driverName, claimUID, baseRequestRef)) } // FileOperations defines optional callbacks for handling CDI files @@ -328,15 +329,20 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drap var devices []Device for _, result := range claim.Status.Allocation.Devices.Results { - requestName := result.Request + // Only handle allocations for the current driver. + if ex.driverName != result.Driver { + continue + } + + baseRequestName := resourceclaim.BaseRequestRef(result.Request) // The driver joins all env variables in the order in which // they appear in results (last one wins). + configs := resourceclaim.ConfigForResult(claim.Status.Allocation.Devices.Config, result) env := make(map[string]string) - for i, config := range claim.Status.Allocation.Devices.Config { - if config.Opaque == nil || - config.Opaque.Driver != ex.driverName || - len(config.Requests) > 0 && !slices.Contains(config.Requests, requestName) { + for i, config := range configs { + // Only use configs for the current driver. + if config.Opaque.Driver != ex.driverName { continue } if err := extractParameters(config.Opaque.Parameters, &env, config.Source == resourceapi.AllocationConfigSourceClass); err != nil { @@ -346,11 +352,11 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drap // It also sets a claim__=true env variable. // This can be used to identify which devices where mapped into a container. - claimReqName := "claim_" + claim.Name + "_" + requestName + claimReqName := "claim_" + claim.Name + "_" + baseRequestName claimReqName = regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(claimReqName, "_") env[claimReqName] = "true" - deviceName := "claim-" + claimReq.UID + "-" + requestName + deviceName := "claim-" + claimReq.UID + "-" + baseRequestName vendor := ex.driverName class := "test" cdiDeviceID := vendor + "/" + class + "=" + deviceName @@ -385,7 +391,7 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drap }, }, } - filePath := ex.getJSONFilePath(claimReq.UID, requestName) + filePath := ex.getJSONFilePath(claimReq.UID, baseRequestName) buffer, err := json.Marshal(spec) if err != nil { return nil, fmt.Errorf("marshal spec: %w", err) @@ -396,7 +402,7 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drap device := Device{ PoolName: result.Pool, DeviceName: result.Device, - RequestName: requestName, + RequestName: baseRequestName, CDIDeviceID: cdiDeviceID, } devices = append(devices, device) diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go index 9df5edabfab..6c9b8e1f5b7 100644 --- a/test/e2e/feature/feature.go +++ b/test/e2e/feature/feature.go @@ -118,6 +118,19 @@ var ( // OWNER: sig-node // Testing downward API huge pages DownwardAPIHugePages = framework.WithFeature(framework.ValidFeatures.Add("DownwardAPIHugePages")) + + // owning-sig: sig-scheduling + // kep: https://kep.k8s.io/4816 + // test-infra jobs: + // - "ci-kind-dra-all" in https://testgrid.k8s.io/sig-node-dynamic-resource-allocation + // + // This label is used for tests which need: + // - the DynamicResourceAllocation *and* DRAPrioritizedList feature gates + // - the resource.k8s.io API group + // - a container runtime where support for CDI (https://github.com/cncf-tags/container-device-interface) + // is enabled such that passing CDI device IDs through CRI fields is supported + DRAPrioritizedList = framework.WithFeature(framework.ValidFeatures.Add("DRAPrioritizedList")) + // owning-sig: sig-node // kep: https://kep.k8s.io/4381 // test-infra jobs: