diff --git a/test/e2e/dra/deploy.go b/test/e2e/dra/deploy.go index 4448ac244de..eb9b424bed2 100644 --- a/test/e2e/dra/deploy.go +++ b/test/e2e/dra/deploy.go @@ -38,7 +38,6 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1alpha3" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -99,6 +98,7 @@ func NewNodes(f *framework.Framework, minNodes, maxNodes int) *Nodes { for _, node := range nodeList.Items { nodes.NodeNames = append(nodes.NodeNames, node.Name) } + sort.Strings(nodes.NodeNames) framework.Logf("testing on nodes %v", nodes.NodeNames) // Watch claims in the namespace. This is useful for monitoring a test @@ -153,7 +153,7 @@ func validateClaim(claim *resourceapi.ResourceClaim) { // NewDriver sets up controller (as client of the cluster) and // kubelet plugin (via proxy) before the test runs. It cleans // up after the test. -func NewDriver(f *framework.Framework, nodes *Nodes, configureResources func() app.Resources) *Driver { +func NewDriver(f *framework.Framework, nodes *Nodes, configureResources func() app.Resources, devicesPerNode ...map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute) *Driver { d := &Driver{ f: f, fail: map[MethodInstance]bool{}, @@ -169,7 +169,7 @@ func NewDriver(f *framework.Framework, nodes *Nodes, configureResources func() a resources.Nodes = nodes.NodeNames } ginkgo.DeferCleanup(d.IsGone) // Register first so it gets called last. - d.SetUp(nodes, resources) + d.SetUp(nodes, resources, devicesPerNode...) ginkgo.DeferCleanup(d.TearDown) }) return d @@ -195,13 +195,8 @@ type Driver struct { // In addition, there is one entry for a fictional node. Nodes map[string]KubeletPlugin - parameterMode parameterMode - parameterAPIGroup string - parameterAPIVersion string - claimParameterAPIKind string - classParameterAPIKind string - - NodeV1alpha3 bool + parameterMode parameterMode // empty == parameterModeStructured + NodeV1alpha3 bool mutex sync.Mutex fail map[MethodInstance]bool @@ -216,12 +211,11 @@ type KubeletPlugin struct { type parameterMode string const ( - parameterModeConfigMap parameterMode = "configmap" // ConfigMap parameters, control plane controller. - parameterModeStructured parameterMode = "structured" // No ConfigMaps, directly create and reference in-tree parameter objects. - parameterModeTranslated parameterMode = "translated" // Reference ConfigMaps in claim and class, generate in-tree parameter objects. + parameterModeClassicDRA parameterMode = "classic" // control plane controller + parameterModeStructured parameterMode = "structured" // allocation through scheduler ) -func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) { +func (d *Driver) SetUp(nodes *Nodes, resources app.Resources, devicesPerNode ...map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute) { ginkgo.By(fmt.Sprintf("deploying driver on nodes %v", nodes.NodeNames)) d.Nodes = make(map[string]KubeletPlugin) d.Name = d.f.UniqueName + d.NameSuffix + ".k8s.io" @@ -236,8 +230,12 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) { d.ctx = ctx d.cleanup = append(d.cleanup, cancel) + if d.parameterMode == "" { + d.parameterMode = parameterModeStructured + } + switch d.parameterMode { - case "", parameterModeConfigMap: + case parameterModeClassicDRA: // The controller is easy: we simply connect to the API server. d.Controller = app.NewController(d.f.ClientSet, resources) d.wg.Add(1) @@ -245,6 +243,49 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) { defer d.wg.Done() d.Controller.Run(d.ctx, 5 /* workers */) }() + case parameterModeStructured: + if !resources.NodeLocal { + // Publish one resource pool with "network-attached" devices. + slice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: d.Name, // globally unique + }, + Spec: resourceapi.ResourceSliceSpec{ + Driver: d.Name, + Pool: resourceapi.ResourcePool{ + Name: "network", + Generation: 1, + ResourceSliceCount: 1, + }, + NodeSelector: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{{ + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: nodes.NodeNames, + }}, + }}, + }, + }, + } + maxAllocations := resources.MaxAllocations + if maxAllocations <= 0 { + // Cannot be empty, otherwise nothing runs. + maxAllocations = 10 + } + for i := 0; i < maxAllocations; i++ { + slice.Spec.Devices = append(slice.Spec.Devices, resourceapi.Device{ + Name: fmt.Sprintf("device-%d", i), + Basic: &resourceapi.BasicDevice{}, + }) + } + + _, err := d.f.ClientSet.ResourceV1alpha3().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func(ctx context.Context) { + framework.ExpectNoError(d.f.ClientSet.ResourceV1alpha3().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{})) + }) + } } manifests := []string{ @@ -252,24 +293,12 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) { // container names, etc.). "test/e2e/testing-manifests/dra/dra-test-driver-proxy.yaml", } - if d.parameterMode == "" { - d.parameterMode = parameterModeConfigMap - } - var numResourceInstances = -1 // disabled - if d.parameterMode != parameterModeConfigMap { - numResourceInstances = resources.MaxAllocations + var numDevices = -1 // disabled + if d.parameterMode != parameterModeClassicDRA && resources.NodeLocal { + numDevices = resources.MaxAllocations } switch d.parameterMode { - case parameterModeConfigMap, parameterModeTranslated: - d.parameterAPIGroup = "" - d.parameterAPIVersion = "v1" - d.claimParameterAPIKind = "ConfigMap" - d.classParameterAPIKind = "ConfigMap" - case parameterModeStructured: - d.parameterAPIGroup = "resource.k8s.io" - d.parameterAPIVersion = "v1alpha3" - d.claimParameterAPIKind = "ResourceClaimParameters" - d.classParameterAPIKind = "ResourceClassParameters" + case parameterModeClassicDRA, parameterModeStructured: default: framework.Failf("unknown test driver parameter mode: %s", d.parameterMode) } @@ -314,10 +343,6 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) { item.Spec.Template.Spec.Volumes[2].HostPath.Path = path.Join(framework.TestContext.KubeletRootDir, "plugins_registry") item.Spec.Template.Spec.Containers[0].Args = append(item.Spec.Template.Spec.Containers[0].Args, "--endpoint=/plugins_registry/"+d.Name+"-reg.sock") item.Spec.Template.Spec.Containers[1].Args = append(item.Spec.Template.Spec.Containers[1].Args, "--endpoint=/dra/"+d.Name+".sock") - case *apiextensionsv1.CustomResourceDefinition: - item.Name = strings.ReplaceAll(item.Name, "dra.e2e.example.com", d.parameterAPIGroup) - item.Spec.Group = d.parameterAPIGroup - } return nil }, manifests...) @@ -336,9 +361,12 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) { pods, err := d.f.ClientSet.CoreV1().Pods(d.f.Namespace.Name).List(ctx, metav1.ListOptions{LabelSelector: selector.String()}) framework.ExpectNoError(err, "list proxy pods") gomega.Expect(numNodes).To(gomega.Equal(int32(len(pods.Items))), "number of proxy pods") + sort.Slice(pods.Items, func(i, j int) bool { + return pods.Items[i].Spec.NodeName < pods.Items[j].Spec.NodeName + }) // Run registrar and plugin for each of the pods. - for _, pod := range pods.Items { + for i, pod := range pods.Items { // Need a local variable, not the loop variable, for the anonymous // callback functions below. pod := pod @@ -361,18 +389,23 @@ func (d *Driver) SetUp(nodes *Nodes, resources app.Resources) { logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "kubelet plugin"), "node", pod.Spec.NodeName, "pod", klog.KObj(&pod)) loggerCtx := klog.NewContext(ctx, logger) - plugin, err := app.StartPlugin(loggerCtx, "/cdi", d.Name, driverClient, nodename, - app.FileOperations{ - Create: func(name string, content []byte) error { - klog.Background().Info("creating CDI file", "node", nodename, "filename", name, "content", string(content)) - return d.createFile(&pod, name, content) - }, - Remove: func(name string) error { - klog.Background().Info("deleting CDI file", "node", nodename, "filename", name) - return d.removeFile(&pod, name) - }, - NumResourceInstances: numResourceInstances, + fileOps := app.FileOperations{ + Create: func(name string, content []byte) error { + klog.Background().Info("creating CDI file", "node", nodename, "filename", name, "content", string(content)) + return d.createFile(&pod, name, content) }, + Remove: func(name string) error { + klog.Background().Info("deleting CDI file", "node", nodename, "filename", name) + return d.removeFile(&pod, name) + }, + } + if i < len(devicesPerNode) { + fileOps.Devices = devicesPerNode[i] + fileOps.NumDevices = -1 + } else { + fileOps.NumDevices = numDevices + } + plugin, err := app.StartPlugin(loggerCtx, "/cdi", d.Name, driverClient, nodename, fileOps, kubeletplugin.GRPCVerbosity(0), kubeletplugin.GRPCInterceptor(func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { return d.interceptor(nodename, ctx, req, info, handler) @@ -527,7 +560,7 @@ func (d *Driver) TearDown() { func (d *Driver) IsGone(ctx context.Context) { gomega.Eventually(ctx, func(ctx context.Context) ([]resourceapi.ResourceSlice, error) { - slices, err := d.f.ClientSet.ResourceV1alpha3().ResourceSlices().List(ctx, metav1.ListOptions{FieldSelector: "driverName=" + d.Name}) + slices, err := d.f.ClientSet.ResourceV1alpha3().ResourceSlices().List(ctx, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + d.Name}) if err != nil { return nil, err } diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index ea3761f06cc..b1367796897 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -18,9 +18,11 @@ package dra import ( "context" - "encoding/json" + _ "embed" "errors" "fmt" + "regexp" + "sort" "strings" "sync" "time" @@ -33,6 +35,7 @@ import ( v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1alpha3" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -80,105 +83,199 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.Context("kubelet", func() { nodes := NewNodes(f, 1, 1) + driver := NewDriver(f, nodes, networkResources) + b := newBuilder(f, driver) - ginkgo.Context("with ConfigMap parameters", func() { - driver := NewDriver(f, nodes, networkResources) - b := newBuilder(f, driver) + ginkgo.It("registers plugin", func() { + ginkgo.By("the driver is running") + }) - ginkgo.It("registers plugin", func() { - ginkgo.By("the driver is running") - }) + ginkgo.It("must retry NodePrepareResources", func(ctx context.Context) { + // We have exactly one host. + m := MethodInstance{driver.Nodenames()[0], NodePrepareResourcesMethod} - ginkgo.It("must retry NodePrepareResources", func(ctx context.Context) { - // We have exactly one host. - m := MethodInstance{driver.Nodenames()[0], NodePrepareResourcesMethod} + driver.Fail(m, true) - driver.Fail(m, true) + ginkgo.By("waiting for container startup to fail") + pod, template := b.podInline() - ginkgo.By("waiting for container startup to fail") - parameters := b.parameters() - pod, template := b.podInline() + b.create(ctx, pod, template) - b.create(ctx, parameters, pod, template) - - ginkgo.By("wait for NodePrepareResources call") - gomega.Eventually(ctx, func(ctx context.Context) error { - if driver.CallCount(m) == 0 { - return errors.New("NodePrepareResources not called yet") - } - return nil - }).WithTimeout(podStartTimeout).Should(gomega.Succeed()) - - ginkgo.By("allowing container startup to succeed") - callCount := driver.CallCount(m) - driver.Fail(m, false) - err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "start pod with inline resource claim") - if driver.CallCount(m) == callCount { - framework.Fail("NodePrepareResources should have been called again") + ginkgo.By("wait for NodePrepareResources call") + gomega.Eventually(ctx, func(ctx context.Context) error { + if driver.CallCount(m) == 0 { + return errors.New("NodePrepareResources not called yet") } - }) + return nil + }).WithTimeout(podStartTimeout).Should(gomega.Succeed()) - ginkgo.It("must not run a pod if a claim is not ready", func(ctx context.Context) { - claim := b.externalClaim() - b.create(ctx, claim) - pod := b.podExternal() + ginkgo.By("allowing container startup to succeed") + callCount := driver.CallCount(m) + driver.Fail(m, false) + err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "start pod with inline resource claim") + if driver.CallCount(m) == callCount { + framework.Fail("NodePrepareResources should have been called again") + } + }) - // This bypasses scheduling and therefore the pod gets - // to run on the node although the claim is not ready. - // Because the parameters are missing, the claim - // also cannot be allocated later. - pod.Spec.NodeName = nodes.NodeNames[0] - b.create(ctx, pod) + ginkgo.It("must not run a pod if a claim is not ready", func(ctx context.Context) { + claim := b.externalClaim() + b.create(ctx, claim) + pod := b.podExternal() - gomega.Consistently(ctx, func(ctx context.Context) error { - testPod, err := b.f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("expected the test pod %s to exist: %w", pod.Name, err) - } - if testPod.Status.Phase != v1.PodPending { - return fmt.Errorf("pod %s: unexpected status %s, expected status: %s", pod.Name, testPod.Status.Phase, v1.PodPending) - } - return nil - }, 20*time.Second, 200*time.Millisecond).Should(gomega.BeNil()) - }) + // This bypasses scheduling and therefore the pod gets + // to run on the node although the claim is not ready. + // Because the parameters are missing, the claim + // also cannot be allocated later. + pod.Spec.NodeName = nodes.NodeNames[0] + b.create(ctx, pod) - ginkgo.It("must unprepare resources for force-deleted pod", func(ctx context.Context) { - parameters := b.parameters() - claim := b.externalClaim() - pod := b.podExternal() - zero := int64(0) - pod.Spec.TerminationGracePeriodSeconds = &zero - - b.create(ctx, parameters, claim, pod) - - b.testPod(ctx, f.ClientSet, pod) - - ginkgo.By(fmt.Sprintf("force delete test pod %s", pod.Name)) - err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &zero}) - if !apierrors.IsNotFound(err) { - framework.ExpectNoError(err, "force delete test pod") + gomega.Consistently(ctx, func(ctx context.Context) error { + testPod, err := b.f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("expected the test pod %s to exist: %w", pod.Name, err) } - - for host, plugin := range b.driver.Nodes { - ginkgo.By(fmt.Sprintf("waiting for resources on %s to be unprepared", host)) - gomega.Eventually(plugin.GetPreparedResources).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host) + if testPod.Status.Phase != v1.PodPending { + return fmt.Errorf("pod %s: unexpected status %s, expected status: %s", pod.Name, testPod.Status.Phase, v1.PodPending) } - }) + return nil + }, 20*time.Second, 200*time.Millisecond).Should(gomega.BeNil()) + }) - ginkgo.It("must skip NodePrepareResource if not used by any container", func(ctx context.Context) { - parameters := b.parameters() - pod, template := b.podInline() - for i := range pod.Spec.Containers { - pod.Spec.Containers[i].Resources.Claims = nil - } - b.create(ctx, parameters, pod, template) - framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod), "start pod") - for host, plugin := range b.driver.Nodes { - gomega.Expect(plugin.GetPreparedResources()).Should(gomega.BeEmpty(), "not claims should be prepared on host %s while pod is running", host) - } - }) + ginkgo.It("must unprepare resources for force-deleted pod", func(ctx context.Context) { + claim := b.externalClaim() + pod := b.podExternal() + zero := int64(0) + pod.Spec.TerminationGracePeriodSeconds = &zero + b.create(ctx, claim, pod) + + b.testPod(ctx, f.ClientSet, pod) + + ginkgo.By(fmt.Sprintf("force delete test pod %s", pod.Name)) + err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &zero}) + if !apierrors.IsNotFound(err) { + framework.ExpectNoError(err, "force delete test pod") + } + + for host, plugin := range b.driver.Nodes { + ginkgo.By(fmt.Sprintf("waiting for resources on %s to be unprepared", host)) + gomega.Eventually(plugin.GetPreparedResources).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host) + } + }) + + ginkgo.It("must call NodePrepareResources even if not used by any container", func(ctx context.Context) { + pod, template := b.podInline() + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].Resources.Claims = nil + } + b.create(ctx, pod, template) + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod), "start pod") + for host, plugin := range b.driver.Nodes { + gomega.Expect(plugin.GetPreparedResources()).ShouldNot(gomega.BeEmpty(), "claims should be prepared on host %s while pod is running", host) + } + }) + + ginkgo.It("must map configs and devices to the right containers", func(ctx context.Context) { + // Several claims, each with three requests and three configs. + // One config applies to all requests, the other two only to one request each. + claimForAllContainers := b.externalClaim() + claimForAllContainers.Name = "all" + claimForAllContainers.Spec.Devices.Requests = append(claimForAllContainers.Spec.Devices.Requests, + *claimForAllContainers.Spec.Devices.Requests[0].DeepCopy(), + *claimForAllContainers.Spec.Devices.Requests[0].DeepCopy(), + ) + claimForAllContainers.Spec.Devices.Requests[0].Name = "req0" + claimForAllContainers.Spec.Devices.Requests[1].Name = "req1" + claimForAllContainers.Spec.Devices.Requests[2].Name = "req2" + claimForAllContainers.Spec.Devices.Config = append(claimForAllContainers.Spec.Devices.Config, + *claimForAllContainers.Spec.Devices.Config[0].DeepCopy(), + *claimForAllContainers.Spec.Devices.Config[0].DeepCopy(), + ) + claimForAllContainers.Spec.Devices.Config[0].Requests = nil + claimForAllContainers.Spec.Devices.Config[1].Requests = []string{"req1"} + claimForAllContainers.Spec.Devices.Config[2].Requests = []string{"req2"} + claimForAllContainers.Spec.Devices.Config[0].Opaque.Parameters.Raw = []byte(`{"all_config0":"true"}`) + claimForAllContainers.Spec.Devices.Config[1].Opaque.Parameters.Raw = []byte(`{"all_config1":"true"}`) + claimForAllContainers.Spec.Devices.Config[2].Opaque.Parameters.Raw = []byte(`{"all_config2":"true"}`) + + claimForContainer0 := claimForAllContainers.DeepCopy() + claimForContainer0.Name = "container0" + claimForContainer0.Spec.Devices.Config[0].Opaque.Parameters.Raw = []byte(`{"container0_config0":"true"}`) + claimForContainer0.Spec.Devices.Config[1].Opaque.Parameters.Raw = []byte(`{"container0_config1":"true"}`) + claimForContainer0.Spec.Devices.Config[2].Opaque.Parameters.Raw = []byte(`{"container0_config2":"true"}`) + claimForContainer1 := claimForAllContainers.DeepCopy() + claimForContainer1.Name = "container1" + claimForContainer1.Spec.Devices.Config[0].Opaque.Parameters.Raw = []byte(`{"container1_config0":"true"}`) + claimForContainer1.Spec.Devices.Config[1].Opaque.Parameters.Raw = []byte(`{"container1_config1":"true"}`) + claimForContainer1.Spec.Devices.Config[2].Opaque.Parameters.Raw = []byte(`{"container1_config2":"true"}`) + + pod := b.podExternal() + pod.Spec.ResourceClaims = []v1.PodResourceClaim{ + { + Name: "all", + ResourceClaimName: &claimForAllContainers.Name, + }, + { + Name: "container0", + ResourceClaimName: &claimForContainer0.Name, + }, + { + Name: "container1", + ResourceClaimName: &claimForContainer1.Name, + }, + } + + // Add a second container. + pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy()) + pod.Spec.Containers[0].Name = "container0" + pod.Spec.Containers[1].Name = "container1" + + // All claims use unique env variables which can be used to verify that they + // have been mapped into the right containers. In addition, the test driver + // also sets "claim__=true" with non-alphanumeric + // replaced by underscore. + + // Both requests (claim_*_req*) and all user configs (user_*_config*). + allContainersEnv := []string{ + "user_all_config0", "true", + "user_all_config1", "true", + "user_all_config2", "true", + "claim_all_req0", "true", + "claim_all_req1", "true", + "claim_all_req2", "true", + } + + // Everything from the "all" claim and everything from the "container0" claim. + pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: "all"}, {Name: "container0"}} + container0Env := []string{ + "user_container0_config0", "true", + "user_container0_config1", "true", + "user_container0_config2", "true", + "claim_container0_req0", "true", + "claim_container0_req1", "true", + "claim_container0_req2", "true", + } + container0Env = append(container0Env, allContainersEnv...) + + // Everything from the "all" claim, but only the second request from the "container1" claim. + // The first two configs apply. + pod.Spec.Containers[1].Resources.Claims = []v1.ResourceClaim{{Name: "all"}, {Name: "container1", Request: "req1"}} + container1Env := []string{ + "user_container1_config0", "true", + "user_container1_config1", "true", + // Does not apply: user_container1_config2 + "claim_container1_req1", "true", + } + container1Env = append(container1Env, allContainersEnv...) + + b.create(ctx, claimForAllContainers, claimForContainer0, claimForContainer1, pod) + err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) + framework.ExpectNoError(err, "start pod") + + testContainerEnv(ctx, f.ClientSet, pod, pod.Spec.Containers[0].Name, true, container0Env...) + testContainerEnv(ctx, f.ClientSet, pod, pod.Spec.Containers[1].Name, true, container1Env...) }) }) @@ -186,81 +283,64 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, // claims, both inline and external. claimTests := func(b *builder, driver *Driver) { ginkgo.It("supports simple pod referencing inline resource claim", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() pod, template := b.podInline() - objects = append(objects, pod, template) - b.create(ctx, objects...) - - b.testPod(ctx, f.ClientSet, pod, expectedEnv...) + b.create(ctx, pod, template) + b.testPod(ctx, f.ClientSet, pod) }) ginkgo.It("supports inline claim referenced by multiple containers", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() pod, template := b.podInlineMultiple() - objects = append(objects, pod, template) - b.create(ctx, objects...) - - b.testPod(ctx, f.ClientSet, pod, expectedEnv...) + b.create(ctx, pod, template) + b.testPod(ctx, f.ClientSet, pod) }) ginkgo.It("supports simple pod referencing external resource claim", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() pod := b.podExternal() claim := b.externalClaim() - objects = append(objects, claim, pod) - b.create(ctx, objects...) - - b.testPod(ctx, f.ClientSet, pod, expectedEnv...) + b.create(ctx, claim, pod) + b.testPod(ctx, f.ClientSet, pod) }) ginkgo.It("supports external claim referenced by multiple pods", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() pod1 := b.podExternal() pod2 := b.podExternal() pod3 := b.podExternal() claim := b.externalClaim() - objects = append(objects, claim, pod1, pod2, pod3) - b.create(ctx, objects...) + b.create(ctx, claim, pod1, pod2, pod3) for _, pod := range []*v1.Pod{pod1, pod2, pod3} { - b.testPod(ctx, f.ClientSet, pod, expectedEnv...) + b.testPod(ctx, f.ClientSet, pod) } }) ginkgo.It("supports external claim referenced by multiple containers of multiple pods", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() pod1 := b.podExternalMultiple() pod2 := b.podExternalMultiple() pod3 := b.podExternalMultiple() claim := b.externalClaim() - objects = append(objects, claim, pod1, pod2, pod3) - b.create(ctx, objects...) + b.create(ctx, claim, pod1, pod2, pod3) for _, pod := range []*v1.Pod{pod1, pod2, pod3} { - b.testPod(ctx, f.ClientSet, pod, expectedEnv...) + b.testPod(ctx, f.ClientSet, pod) } }) ginkgo.It("supports init containers", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() pod, template := b.podInline() pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]} pod.Spec.InitContainers[0].Name += "-init" // This must succeed for the pod to start. pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env | grep user_a=b"} - objects = append(objects, pod, template) - b.create(ctx, objects...) + b.create(ctx, pod, template) - b.testPod(ctx, f.ClientSet, pod, expectedEnv...) + b.testPod(ctx, f.ClientSet, pod) }) ginkgo.It("removes reservation from claim when pod is done", func(ctx context.Context) { - objects, _ := b.flexibleParameters() pod := b.podExternal() claim := b.externalClaim() pod.Spec.Containers[0].Command = []string{"true"} - objects = append(objects, claim, pod) - b.create(ctx, objects...) + b.create(ctx, claim, pod) ginkgo.By("waiting for pod to finish") framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish") @@ -271,11 +351,9 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) ginkgo.It("deletes generated claims when pod is done", func(ctx context.Context) { - objects, _ := b.flexibleParameters() pod, template := b.podInline() pod.Spec.Containers[0].Command = []string{"true"} - objects = append(objects, template, pod) - b.create(ctx, objects...) + b.create(ctx, template, pod) ginkgo.By("waiting for pod to finish") framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish") @@ -290,12 +368,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) ginkgo.It("does not delete generated claims when pod is restarting", func(ctx context.Context) { - objects, _ := b.flexibleParameters() pod, template := b.podInline() pod.Spec.Containers[0].Command = []string{"sh", "-c", "sleep 1; exit 1"} pod.Spec.RestartPolicy = v1.RestartPolicyAlways - objects = append(objects, template, pod) - b.create(ctx, objects...) + b.create(ctx, template, pod) ginkgo.By("waiting for pod to restart twice") gomega.Eventually(ctx, func(ctx context.Context) (*v1.Pod, error) { @@ -307,17 +383,15 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) ginkgo.It("must deallocate after use", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() pod := b.podExternal() claim := b.externalClaim() - objects = append(objects, claim, pod) - b.create(ctx, objects...) + b.create(ctx, claim, pod) gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) { return b.f.ClientSet.ResourceV1alpha3().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) }).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil))) - b.testPod(ctx, f.ClientSet, pod, expectedEnv...) + b.testPod(ctx, f.ClientSet, pod) ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(pod))) framework.ExpectNoError(b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{})) @@ -340,23 +414,20 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, driver := NewDriver(f, nodes, generateResources) // All tests get their own driver instance. driver.parameterMode = parameterMode b := newBuilder(f, driver) - // We need the parameters name *before* creating it. - b.parametersCounter = 1 - b.classParametersName = b.parametersName() + // We have to set the parameters *before* creating the class. + b.classParameters = `{"x":"y"}` + expectedEnv := []string{"admin_x", "y"} + _, expected := b.parametersEnv() + expectedEnv = append(expectedEnv, expected...) ginkgo.It("supports claim and class parameters", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() - pod, template := b.podInline() - objects = append(objects, pod, template) - - b.create(ctx, objects...) - + b.create(ctx, pod, template) b.testPod(ctx, f.ClientSet, pod, expectedEnv...) }) ginkgo.It("supports reusing resources", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() + var objects []klog.KMetadata pods := make([]*v1.Pod, numPods) for i := 0; i < numPods; i++ { pod, template := b.podInline() @@ -384,9 +455,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) ginkgo.It("supports sharing a claim concurrently", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() + var objects []klog.KMetadata objects = append(objects, b.externalClaim()) - pods := make([]*v1.Pod, numPods) for i := 0; i < numPods; i++ { pod := b.podExternal() @@ -412,7 +482,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) f.It("supports sharing a claim sequentially", f.WithSlow(), func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() + var objects []klog.KMetadata objects = append(objects, b.externalClaim()) // This test used to test usage of the claim by one pod @@ -448,12 +518,14 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, wg.Wait() }) - ginkgo.It("retries pod scheduling after creating resource class", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() + ginkgo.It("retries pod scheduling after creating device class", func(ctx context.Context) { + var objects []klog.KMetadata pod, template := b.podInline() - class, err := f.ClientSet.ResourceV1alpha3().ResourceClasses().Get(ctx, template.Spec.Spec.ResourceClassName, metav1.GetOptions{}) + deviceClassName := template.Spec.Spec.Devices.Requests[0].DeviceClassName + class, err := f.ClientSet.ResourceV1alpha3().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{}) framework.ExpectNoError(err) - template.Spec.Spec.ResourceClassName += "-b" + deviceClassName += "-b" + template.Spec.Spec.Devices.Requests[0].DeviceClassName = deviceClassName objects = append(objects, template, pod) b.create(ctx, objects...) @@ -461,33 +533,46 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, class.UID = "" class.ResourceVersion = "" - class.Name = template.Spec.Spec.ResourceClassName + class.Name = deviceClassName b.create(ctx, class) b.testPod(ctx, f.ClientSet, pod, expectedEnv...) }) - ginkgo.It("retries pod scheduling after updating resource class", func(ctx context.Context) { - objects, expectedEnv := b.flexibleParameters() + ginkgo.It("retries pod scheduling after updating device class", func(ctx context.Context) { + var objects []klog.KMetadata pod, template := b.podInline() - // First modify the class so that it matches no nodes. - class, err := f.ClientSet.ResourceV1alpha3().ResourceClasses().Get(ctx, template.Spec.Spec.ResourceClassName, metav1.GetOptions{}) + // First modify the class so that it matches no nodes (for classic DRA) and no devices (structured parameters). + deviceClassName := template.Spec.Spec.Devices.Requests[0].DeviceClassName + class, err := f.ClientSet.ResourceV1alpha3().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{}) framework.ExpectNoError(err) - class.SuitableNodes = &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "no-such-label", - Operator: v1.NodeSelectorOpIn, - Values: []string{"no-such-value"}, + originalClass := class.DeepCopy() + switch driver.parameterMode { + case parameterModeClassicDRA: + class.Spec.SuitableNodes = &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "no-such-label", + Operator: v1.NodeSelectorOpIn, + Values: []string{"no-such-value"}, + }, }, }, }, - }, + } + case parameterModeStructured: + class.Spec.Selectors = []resourceapi.DeviceSelector{{ + CEL: &resourceapi.CELDeviceSelector{ + Expression: "false", + }, + }} + default: + framework.Failf("unexpected mode: %s", driver.parameterMode) } - class, err = f.ClientSet.ResourceV1alpha3().ResourceClasses().Update(ctx, class, metav1.UpdateOptions{}) + class, err = f.ClientSet.ResourceV1alpha3().DeviceClasses().Update(ctx, class, metav1.UpdateOptions{}) framework.ExpectNoError(err) // Now create the pod. @@ -497,8 +582,9 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace)) // Unblock the pod. - class.SuitableNodes = nil - _, err = f.ClientSet.ResourceV1alpha3().ResourceClasses().Update(ctx, class, metav1.UpdateOptions{}) + class.Spec.SuitableNodes = originalClass.Spec.SuitableNodes + class.Spec.Selectors = originalClass.Spec.Selectors + _, err = f.ClientSet.ResourceV1alpha3().DeviceClasses().Update(ctx, class, metav1.UpdateOptions{}) framework.ExpectNoError(err) b.testPod(ctx, f.ClientSet, pod, expectedEnv...) @@ -528,10 +614,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, // These tests depend on having more than one node and a DRA driver controller. multiNodeDRAControllerTests := func(nodes *Nodes) { driver := NewDriver(f, nodes, networkResources) + driver.parameterMode = parameterModeClassicDRA b := newBuilder(f, driver) ginkgo.It("schedules onto different nodes", func(ctx context.Context) { - parameters := b.parameters() label := "app.kubernetes.io/instance" instance := f.UniqueName + "-test-app" antiAffinity := &v1.Affinity{ @@ -557,7 +643,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, pod1 := createPod() pod2 := createPod() claim := b.externalClaim() - b.create(ctx, parameters, claim, pod1, pod2) + b.create(ctx, claim, pod1, pod2) for _, pod := range []*v1.Pod{pod1, pod2} { err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) @@ -572,13 +658,12 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, // nodes by running `docker stop `, which is very kind-specific. f.It(f.WithSerial(), f.WithDisruptive(), f.WithSlow(), "must deallocate on non graceful node shutdown", func(ctx context.Context) { ginkgo.By("create test pod") - parameters := b.parameters() label := "app.kubernetes.io/instance" instance := f.UniqueName + "-test-app" pod := b.podExternal() pod.Labels[label] = instance claim := b.externalClaim() - b.create(ctx, parameters, claim, pod) + b.create(ctx, claim, pod) ginkgo.By("wait for test pod " + pod.Name + " to run") labelSelector := labels.SelectorFromSet(labels.Set(pod.Labels)) @@ -617,7 +702,75 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, multiNodeTests := func(parameterMode parameterMode) { nodes := NewNodes(f, 2, 8) - if parameterMode == parameterModeConfigMap { + switch parameterMode { + case parameterModeStructured: + ginkgo.Context("with different ResourceSlices", func() { + firstDevice := "pre-defined-device-01" + secondDevice := "pre-defined-device-02" + devicesPerNode := []map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + // First node: + { + firstDevice: { + "healthy": {BoolValue: ptr.To(true)}, + "exists": {BoolValue: ptr.To(true)}, + }, + }, + // Second node: + { + secondDevice: { + "healthy": {BoolValue: ptr.To(false)}, + // Has no "exists" attribute! + }, + }, + } + driver := NewDriver(f, nodes, perNode(-1, nodes), devicesPerNode...) + b := newBuilder(f, driver) + + ginkgo.It("keeps pod pending because of CEL runtime errors", func(ctx context.Context) { + // When pod scheduling encounters CEL runtime errors for some nodes, but not all, + // it should still not schedule the pod because there is something wrong with it. + // Scheduling it would make it harder to detect that there is a problem. + // + // This matches the "CEL-runtime-error-for-subset-of-nodes" unit test, except that + // here we try it in combination with the actual scheduler and can extend it with + // other checks, like event handling (future extension). + + gomega.Eventually(ctx, framework.ListObjects(f.ClientSet.ResourceV1alpha3().ResourceSlices().List, + metav1.ListOptions{ + FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driver.Name, + }, + )).Should(gomega.HaveField("Items", gomega.ConsistOf( + gomega.HaveField("Spec.Devices", gomega.ConsistOf( + gomega.Equal(resourceapi.Device{ + Name: firstDevice, + Basic: &resourceapi.BasicDevice{ + Attributes: devicesPerNode[0][firstDevice], + }, + }))), + gomega.HaveField("Spec.Devices", gomega.ConsistOf( + gomega.Equal(resourceapi.Device{ + Name: secondDevice, + Basic: &resourceapi.BasicDevice{ + Attributes: devicesPerNode[1][secondDevice], + }, + }))), + ))) + + pod, template := b.podInline() + template.Spec.Spec.Devices.Requests[0].Selectors = append(template.Spec.Spec.Devices.Requests[0].Selectors, + resourceapi.DeviceSelector{ + CEL: &resourceapi.CELDeviceSelector{ + // Runtime error on one node, but not all. + Expression: fmt.Sprintf(`device.attributes["%s"].exists`, driver.Name), + }, + }, + ) + b.create(ctx, pod, template) + + framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "pod must not get scheduled because of a CEL runtime error") + }) + }) + case parameterModeClassicDRA: ginkgo.Context("with network-attached resources", func() { multiNodeDRAControllerTests(nodes) }) @@ -625,6 +778,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.Context("reallocation", func() { var allocateWrapper2 app.AllocateWrapperType driver := NewDriver(f, nodes, perNode(1, nodes)) + driver.parameterMode = parameterModeClassicDRA driver2 := NewDriver(f, nodes, func() app.Resources { return app.Resources{ NodeLocal: true, @@ -645,6 +799,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, } }) driver2.NameSuffix = "-other" + driver2.parameterMode = parameterModeClassicDRA b := newBuilder(f, driver) b2 := newBuilder(f, driver2) @@ -664,8 +819,6 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ctx, cancel := context.WithCancel(ctx) defer cancel() - parameters1 := b.parameters() - parameters2 := b2.parameters() // Order is relevant here: each pod must be matched with its own claim. pod1claim1 := b.externalClaim() pod1 := b.podExternal() @@ -698,7 +851,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, handler(ctx, claimAllocations, selectedNode) } - b.create(ctx, parameters1, parameters2, pod1claim1, pod1claim2, pod1) + b.create(ctx, pod1claim1, pod1claim2, pod1) ginkgo.By("waiting for one claim from driver1 to be allocated") var nodeSelector *v1.NodeSelector @@ -711,7 +864,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, for _, claim := range claims.Items { if claim.Status.Allocation != nil { allocated++ - nodeSelector = claim.Status.Allocation.AvailableOnNodes + nodeSelector = claim.Status.Allocation.NodeSelector } } return allocated, nil @@ -755,7 +908,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, b := newBuilder(f, driver) ginkgo.It("uses all resources", func(ctx context.Context) { - objs, _ := b.flexibleParameters() + var objs []klog.KMetadata var pods []*v1.Pod for i := 0; i < len(nodes.NodeNames); i++ { pod, template := b.podInline() @@ -802,8 +955,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) } - ginkgo.Context("with ConfigMap parameters", func() { tests(parameterModeConfigMap) }) - ginkgo.Context("with translated parameters", func() { tests(parameterModeTranslated) }) + ginkgo.Context("with classic DRA", func() { tests(parameterModeClassicDRA) }) ginkgo.Context("with structured parameters", func() { tests(parameterModeStructured) }) // TODO (https://github.com/kubernetes/kubernetes/issues/123699): move most of the test below into `testDriver` so that they get @@ -815,15 +967,60 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, b := newBuilder(f, driver) ginkgo.It("truncates the name of a generated resource claim", func(ctx context.Context) { - parameters := b.parameters() pod, template := b.podInline() pod.Name = strings.Repeat("p", 63) pod.Spec.ResourceClaims[0].Name = strings.Repeat("c", 63) pod.Spec.Containers[0].Resources.Claims[0].Name = pod.Spec.ResourceClaims[0].Name - b.create(ctx, parameters, template, pod) + b.create(ctx, template, pod) b.testPod(ctx, f.ClientSet, pod) }) + + ginkgo.It("supports count/resourceclaim.resource ResourceQuota", func(ctx context.Context) { + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "claim-0", + Namespace: f.Namespace.Name, + }, + Spec: resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{{ + Name: "req-0", + DeviceClassName: "my-class", + }}, + }, + }, + } + _, err := f.ClientSet.ResourceV1alpha3().ResourceClaims(f.Namespace.Name).Create(ctx, claim, metav1.CreateOptions{}) + framework.ExpectNoError(err, "create first claim") + + resourceName := "count/resourceclaims.resource.k8s.io" + quota := &v1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "object-count", + Namespace: f.Namespace.Name, + }, + Spec: v1.ResourceQuotaSpec{ + Hard: v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")}, + }, + } + quota, err = f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Create(ctx, quota, metav1.CreateOptions{}) + framework.ExpectNoError(err, "create resource quota") + + // Eventually the quota status should consider the existing claim. + gomega.Eventually(ctx, framework.GetObject(f.ClientSet.CoreV1().ResourceQuotas(quota.Namespace).Get, quota.Name, metav1.GetOptions{})). + Should(gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Status": gomega.Equal(v1.ResourceQuotaStatus{ + Hard: v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")}, + Used: v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")}, + })}))) + + // Now creating another claim should fail. + claim2 := claim.DeepCopy() + claim2.Name = "claim-1" + _, err = f.ClientSet.ResourceV1alpha3().ResourceClaims(f.Namespace.Name).Create(ctx, claim2, metav1.CreateOptions{}) + gomega.Expect(err).Should(gomega.MatchError(gomega.ContainSubstring("exceeded quota: object-count, requested: count/resourceclaims.resource.k8s.io=1, used: count/resourceclaims.resource.k8s.io=1, limited: count/resourceclaims.resource.k8s.io=1")), "creating second claim not allowed") + }) }) // The following tests are all about behavior in combination with a @@ -831,157 +1028,6 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.Context("cluster with DRA driver controller", func() { nodes := NewNodes(f, 1, 4) - ginkgo.Context("with structured parameters", func() { - driver := NewDriver(f, nodes, perNode(1, nodes)) - driver.parameterMode = parameterModeStructured - - f.It("must apply per-node permission checks", func(ctx context.Context) { - // All of the operations use the client set of a kubelet plugin for - // a fictional node which both don't exist, so nothing interferes - // when we actually manage to create a slice. - fictionalNodeName := "dra-fictional-node" - gomega.Expect(nodes.NodeNames).NotTo(gomega.ContainElement(fictionalNodeName)) - fictionalNodeClient := driver.impersonateKubeletPlugin(&v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: fictionalNodeName + "-dra-plugin", - Namespace: f.Namespace.Name, - UID: "12345", - }, - Spec: v1.PodSpec{ - NodeName: fictionalNodeName, - }, - }) - - // This is for some actual node in the cluster. - realNodeName := nodes.NodeNames[0] - realNodeClient := driver.Nodes[realNodeName].ClientSet - - // This is the slice that we try to create. It needs to be deleted - // after testing, if it still exists at that time. - fictionalNodeSlice := &resourceapi.ResourceSlice{ - ObjectMeta: metav1.ObjectMeta{ - Name: fictionalNodeName + "-slice", - }, - NodeName: fictionalNodeName, - DriverName: "dra.example.com", - ResourceModel: resourceapi.ResourceModel{ - NamedResources: &resourceapi.NamedResourcesResources{}, - }, - } - ginkgo.DeferCleanup(func(ctx context.Context) { - err := f.ClientSet.ResourceV1alpha3().ResourceSlices().Delete(ctx, fictionalNodeSlice.Name, metav1.DeleteOptions{}) - if !apierrors.IsNotFound(err) { - framework.ExpectNoError(err) - } - }) - - // Message from test-driver/deploy/example/plugin-permissions.yaml - matchVAPDeniedError := gomega.MatchError(gomega.ContainSubstring("may only modify resourceslices that belong to the node the pod is running on")) - - mustCreate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) *resourceapi.ResourceSlice { - ginkgo.GinkgoHelper() - slice, err := clientSet.ResourceV1alpha3().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}) - framework.ExpectNoError(err, fmt.Sprintf("CREATE: %s + %s", clientName, slice.Name)) - return slice - } - mustUpdate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) *resourceapi.ResourceSlice { - ginkgo.GinkgoHelper() - slice, err := clientSet.ResourceV1alpha3().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{}) - framework.ExpectNoError(err, fmt.Sprintf("UPDATE: %s + %s", clientName, slice.Name)) - return slice - } - mustDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) { - ginkgo.GinkgoHelper() - err := clientSet.ResourceV1alpha3().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err, fmt.Sprintf("DELETE: %s + %s", clientName, slice.Name)) - } - mustFailToCreate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) { - ginkgo.GinkgoHelper() - _, err := clientSet.ResourceV1alpha3().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}) - gomega.Expect(err).To(matchError, fmt.Sprintf("CREATE: %s + %s", clientName, slice.Name)) - } - mustFailToUpdate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) { - ginkgo.GinkgoHelper() - _, err := clientSet.ResourceV1alpha3().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{}) - gomega.Expect(err).To(matchError, fmt.Sprintf("UPDATE: %s + %s", clientName, slice.Name)) - } - mustFailToDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) { - ginkgo.GinkgoHelper() - err := clientSet.ResourceV1alpha3().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{}) - gomega.Expect(err).To(matchError, fmt.Sprintf("DELETE: %s + %s", clientName, slice.Name)) - } - - // Create with different clients, keep it in the end. - mustFailToCreate(realNodeClient, "real plugin", fictionalNodeSlice, matchVAPDeniedError) - createdFictionalNodeSlice := mustCreate(fictionalNodeClient, "fictional plugin", fictionalNodeSlice) - - // Update with different clients. - mustFailToUpdate(realNodeClient, "real plugin", createdFictionalNodeSlice, matchVAPDeniedError) - createdFictionalNodeSlice = mustUpdate(fictionalNodeClient, "fictional plugin", createdFictionalNodeSlice) - createdFictionalNodeSlice = mustUpdate(f.ClientSet, "admin", createdFictionalNodeSlice) - - // Delete with different clients. - mustFailToDelete(realNodeClient, "real plugin", createdFictionalNodeSlice, matchVAPDeniedError) - mustDelete(fictionalNodeClient, "fictional plugin", createdFictionalNodeSlice) - }) - - f.It("must manage ResourceSlices", f.WithSlow(), func(ctx context.Context) { - driverName := driver.Name - - // Now check for exactly the right set of objects for all nodes. - ginkgo.By("check if ResourceSlice object(s) exist on the API server") - resourceClient := f.ClientSet.ResourceV1alpha3().ResourceSlices() - var expectedObjects []any - for _, nodeName := range nodes.NodeNames { - node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - framework.ExpectNoError(err, "get node") - expectedObjects = append(expectedObjects, - gstruct.MatchAllFields(gstruct.Fields{ - "TypeMeta": gstruct.Ignore(), - "ObjectMeta": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "OwnerReferences": gomega.ContainElements( - gstruct.MatchAllFields(gstruct.Fields{ - "APIVersion": gomega.Equal("v1"), - "Kind": gomega.Equal("Node"), - "Name": gomega.Equal(nodeName), - "UID": gomega.Equal(node.UID), - "Controller": gomega.Equal(ptr.To(true)), - "BlockOwnerDeletion": gomega.BeNil(), - }), - ), - }), - "NodeName": gomega.Equal(nodeName), - "DriverName": gomega.Equal(driver.Name), - "ResourceModel": gomega.Equal(resourceapi.ResourceModel{NamedResources: &resourceapi.NamedResourcesResources{ - Instances: []resourceapi.NamedResourcesInstance{{Name: "instance-00"}}, - }}), - }), - ) - } - matchSlices := gomega.ContainElements(expectedObjects...) - getSlices := func(ctx context.Context) ([]resourceapi.ResourceSlice, error) { - slices, err := resourceClient.List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("driverName=%s", driverName)}) - if err != nil { - return nil, err - } - return slices.Items, nil - } - gomega.Eventually(ctx, getSlices).WithTimeout(20 * time.Second).Should(matchSlices) - gomega.Consistently(ctx, getSlices).WithTimeout(20 * time.Second).Should(matchSlices) - - // Removal of node resource slice is tested by the general driver removal code. - }) - - // TODO (https://github.com/kubernetes/kubernetes/issues/123699): more test scenarios: - // - driver returns "unimplemented" as method response - // - driver returns "Unimplemented" as part of stream - // - driver returns EOF - // - driver changes resources - // - // None of those matter if the publishing gets moved into the driver itself, - // which is the goal for 1.31 to support version skew for kubelet. - }) - // kube-controller-manager can trigger delayed allocation for pods where the // node name was already selected when creating the pod. For immediate // allocation, the creator has to ensure that the node matches the claims. @@ -990,20 +1036,18 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, // on all nodes. preScheduledTests := func(b *builder, driver *Driver) { ginkgo.It("supports scheduled pod referencing inline resource claim", func(ctx context.Context) { - parameters := b.parameters() pod, template := b.podInline() pod.Spec.NodeName = nodes.NodeNames[0] - b.create(ctx, parameters, pod, template) + b.create(ctx, pod, template) b.testPod(ctx, f.ClientSet, pod) }) ginkgo.It("supports scheduled pod referencing external resource claim", func(ctx context.Context) { - parameters := b.parameters() claim := b.externalClaim() pod := b.podExternal() pod.Spec.NodeName = nodes.NodeNames[0] - b.create(ctx, parameters, claim, pod) + b.create(ctx, claim, pod) b.testPod(ctx, f.ClientSet, pod) }) @@ -1011,6 +1055,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.Context("with setting ReservedFor", func() { driver := NewDriver(f, nodes, networkResources) + driver.parameterMode = parameterModeClassicDRA b := newBuilder(f, driver) preScheduledTests(b, driver) claimTests(b, driver) @@ -1022,12 +1067,206 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, resources.DontSetReservedFor = true return resources }) + driver.parameterMode = parameterModeClassicDRA b := newBuilder(f, driver) preScheduledTests(b, driver) claimTests(b, driver) }) }) + ginkgo.Context("cluster with structured parameters", func() { + nodes := NewNodes(f, 1, 4) + driver := NewDriver(f, nodes, perNode(1, nodes)) + + f.It("must apply per-node permission checks", func(ctx context.Context) { + // All of the operations use the client set of a kubelet plugin for + // a fictional node which both don't exist, so nothing interferes + // when we actually manage to create a slice. + fictionalNodeName := "dra-fictional-node" + gomega.Expect(nodes.NodeNames).NotTo(gomega.ContainElement(fictionalNodeName)) + fictionalNodeClient := driver.impersonateKubeletPlugin(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fictionalNodeName + "-dra-plugin", + Namespace: f.Namespace.Name, + UID: "12345", + }, + Spec: v1.PodSpec{ + NodeName: fictionalNodeName, + }, + }) + + // This is for some actual node in the cluster. + realNodeName := nodes.NodeNames[0] + realNodeClient := driver.Nodes[realNodeName].ClientSet + + // This is the slice that we try to create. It needs to be deleted + // after testing, if it still exists at that time. + fictionalNodeSlice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: fictionalNodeName + "-slice", + }, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: fictionalNodeName, + Driver: "dra.example.com", + Pool: resourceapi.ResourcePool{ + Name: "some-pool", + ResourceSliceCount: 1, + }, + }, + } + ginkgo.DeferCleanup(func(ctx context.Context) { + err := f.ClientSet.ResourceV1alpha3().ResourceSlices().Delete(ctx, fictionalNodeSlice.Name, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(err) { + framework.ExpectNoError(err) + } + }) + + // Messages from test-driver/deploy/example/plugin-permissions.yaml + matchVAPDeniedError := gomega.MatchError(gomega.ContainSubstring("may only modify resourceslices that belong to the node the pod is running on")) + mustCreate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) *resourceapi.ResourceSlice { + ginkgo.GinkgoHelper() + slice, err := clientSet.ResourceV1alpha3().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), fmt.Sprintf("CREATE: %s + %s", clientName, slice.Name)) + return slice + } + mustUpdate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) *resourceapi.ResourceSlice { + ginkgo.GinkgoHelper() + slice, err := clientSet.ResourceV1alpha3().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), fmt.Sprintf("UPDATE: %s + %s", clientName, slice.Name)) + return slice + } + mustDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) { + ginkgo.GinkgoHelper() + err := clientSet.ResourceV1alpha3().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), fmt.Sprintf("DELETE: %s + %s", clientName, slice.Name)) + } + mustCreateAndDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) { + ginkgo.GinkgoHelper() + slice = mustCreate(clientSet, clientName, slice) + mustDelete(clientSet, clientName, slice) + } + mustFailToCreate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) { + ginkgo.GinkgoHelper() + _, err := clientSet.ResourceV1alpha3().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}) + gomega.Expect(err).To(matchError, fmt.Sprintf("CREATE: %s + %s", clientName, slice.Name)) + } + mustFailToUpdate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) { + ginkgo.GinkgoHelper() + _, err := clientSet.ResourceV1alpha3().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{}) + gomega.Expect(err).To(matchError, fmt.Sprintf("UPDATE: %s + %s", clientName, slice.Name)) + } + mustFailToDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) { + ginkgo.GinkgoHelper() + err := clientSet.ResourceV1alpha3().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(matchError, fmt.Sprintf("DELETE: %s + %s", clientName, slice.Name)) + } + + // Create with different clients, keep it in the end. + mustFailToCreate(realNodeClient, "real plugin", fictionalNodeSlice, matchVAPDeniedError) + mustCreateAndDelete(fictionalNodeClient, "fictional plugin", fictionalNodeSlice) + createdFictionalNodeSlice := mustCreate(f.ClientSet, "admin", fictionalNodeSlice) + + // Update with different clients. + mustFailToUpdate(realNodeClient, "real plugin", createdFictionalNodeSlice, matchVAPDeniedError) + createdFictionalNodeSlice = mustUpdate(fictionalNodeClient, "fictional plugin", createdFictionalNodeSlice) + createdFictionalNodeSlice = mustUpdate(f.ClientSet, "admin", createdFictionalNodeSlice) + + // Delete with different clients. + mustFailToDelete(realNodeClient, "real plugin", createdFictionalNodeSlice, matchVAPDeniedError) + mustDelete(fictionalNodeClient, "fictional plugin", createdFictionalNodeSlice) + + // Now the same for a slice which is not associated with a node. + clusterSlice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-slice", + }, + Spec: resourceapi.ResourceSliceSpec{ + AllNodes: true, + Driver: "another.example.com", + Pool: resourceapi.ResourcePool{ + Name: "cluster-pool", + ResourceSliceCount: 1, + }, + }, + } + ginkgo.DeferCleanup(func(ctx context.Context) { + err := f.ClientSet.ResourceV1alpha3().ResourceSlices().Delete(ctx, clusterSlice.Name, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(err) { + framework.ExpectNoError(err) + } + }) + + // Create with different clients, keep it in the end. + mustFailToCreate(realNodeClient, "real plugin", clusterSlice, matchVAPDeniedError) + mustFailToCreate(fictionalNodeClient, "fictional plugin", clusterSlice, matchVAPDeniedError) + createdClusterSlice := mustCreate(f.ClientSet, "admin", clusterSlice) + + // Update with different clients. + mustFailToUpdate(realNodeClient, "real plugin", createdClusterSlice, matchVAPDeniedError) + mustFailToUpdate(fictionalNodeClient, "fictional plugin", createdClusterSlice, matchVAPDeniedError) + createdClusterSlice = mustUpdate(f.ClientSet, "admin", createdClusterSlice) + + // Delete with different clients. + mustFailToDelete(realNodeClient, "real plugin", createdClusterSlice, matchVAPDeniedError) + mustFailToDelete(fictionalNodeClient, "fictional plugin", createdClusterSlice, matchVAPDeniedError) + mustDelete(f.ClientSet, "admin", createdClusterSlice) + }) + + f.It("must manage ResourceSlices", f.WithSlow(), func(ctx context.Context) { + driverName := driver.Name + + // Now check for exactly the right set of objects for all nodes. + ginkgo.By("check if ResourceSlice object(s) exist on the API server") + resourceClient := f.ClientSet.ResourceV1alpha3().ResourceSlices() + var expectedObjects []any + for _, nodeName := range nodes.NodeNames { + node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err, "get node") + expectedObjects = append(expectedObjects, + gstruct.MatchAllFields(gstruct.Fields{ + "TypeMeta": gstruct.Ignore(), + "ObjectMeta": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "OwnerReferences": gomega.ContainElements( + gstruct.MatchAllFields(gstruct.Fields{ + "APIVersion": gomega.Equal("v1"), + "Kind": gomega.Equal("Node"), + "Name": gomega.Equal(nodeName), + "UID": gomega.Equal(node.UID), + "Controller": gomega.Equal(ptr.To(true)), + "BlockOwnerDeletion": gomega.BeNil(), + }), + ), + }), + "Spec": gstruct.MatchAllFields(gstruct.Fields{ + "Driver": gomega.Equal(driver.Name), + "NodeName": gomega.Equal(nodeName), + "NodeSelector": gomega.BeNil(), + "AllNodes": gomega.BeFalseBecause("slice should be using NodeName"), + "Pool": gstruct.MatchAllFields(gstruct.Fields{ + "Name": gomega.Equal(nodeName), + "Generation": gstruct.Ignore(), + "ResourceSliceCount": gomega.Equal(int64(1)), + }), + "Devices": gomega.Equal([]resourceapi.Device{{Name: "device-00", Basic: &resourceapi.BasicDevice{}}}), + }), + }), + ) + } + matchSlices := gomega.ContainElements(expectedObjects...) + getSlices := func(ctx context.Context) ([]resourceapi.ResourceSlice, error) { + slices, err := resourceClient.List(ctx, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName}) + if err != nil { + return nil, err + } + return slices.Items, nil + } + gomega.Eventually(ctx, getSlices).WithTimeout(20 * time.Second).Should(matchSlices) + gomega.Consistently(ctx, getSlices).WithTimeout(20 * time.Second).Should(matchSlices) + + // Removal of node resource slice is tested by the general driver removal code. + }) + }) + multipleDrivers := func(nodeV1alpha3 bool) { nodes := NewNodes(f, 1, 4) driver1 := NewDriver(f, nodes, perNode(2, nodes)) @@ -1040,8 +1279,6 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, b2 := newBuilder(f, driver2) ginkgo.It("work", func(ctx context.Context) { - parameters1 := b1.parameters() - parameters2 := b2.parameters() claim1 := b1.externalClaim() claim1b := b1.externalClaim() claim2 := b2.externalClaim() @@ -1056,7 +1293,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }, ) } - b1.create(ctx, parameters1, parameters2, claim1, claim1b, claim2, claim2b, pod) + b1.create(ctx, claim1, claim1b, claim2, claim2b, pod) b1.testPod(ctx, f.ClientSet, pod) }) } @@ -1077,36 +1314,43 @@ type builder struct { f *framework.Framework driver *Driver - podCounter int - parametersCounter int - claimCounter int - - classParametersName string + podCounter int + claimCounter int + classParameters string // JSON } -// className returns the default resource class name. +// className returns the default device class name. func (b *builder) className() string { return b.f.UniqueName + b.driver.NameSuffix + "-class" } -// class returns the resource class that the builder's other objects +// class returns the device class that the builder's other objects // reference. -func (b *builder) class() *resourceapi.ResourceClass { - class := &resourceapi.ResourceClass{ +func (b *builder) class() *resourceapi.DeviceClass { + class := &resourceapi.DeviceClass{ ObjectMeta: metav1.ObjectMeta{ Name: b.className(), }, - DriverName: b.driver.Name, - SuitableNodes: b.nodeSelector(), - StructuredParameters: ptr.To(b.driver.parameterMode != parameterModeConfigMap), } - if b.classParametersName != "" { - class.ParametersRef = &resourceapi.ResourceClassParametersReference{ - APIGroup: b.driver.parameterAPIGroup, - Kind: b.driver.classParameterAPIKind, - Name: b.classParametersName, - Namespace: b.f.Namespace.Name, - } + switch b.driver.parameterMode { + case parameterModeClassicDRA: + class.Spec.SuitableNodes = b.nodeSelector() + case parameterModeStructured: + class.Spec.Selectors = []resourceapi.DeviceSelector{{ + CEL: &resourceapi.CELDeviceSelector{ + Expression: fmt.Sprintf(`device.driver == "%s"`, b.driver.Name), + }, + }} + } + if b.classParameters != "" { + class.Spec.Config = []resourceapi.DeviceClassConfiguration{{ + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b.driver.Name, + Parameters: runtime.RawExtension{Raw: []byte(b.classParameters)}, + }, + }, + }} } return class } @@ -1141,158 +1385,44 @@ func (b *builder) externalClaim() *resourceapi.ResourceClaim { ObjectMeta: metav1.ObjectMeta{ Name: name, }, - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: b.className(), - ParametersRef: &resourceapi.ResourceClaimParametersReference{ - APIGroup: b.driver.parameterAPIGroup, - Kind: b.driver.claimParameterAPIKind, - Name: b.parametersName(), - }, - }, + Spec: b.claimSpec(), } } -// flexibleParameters returns parameter objects for claims and -// class with their type depending on the current parameter mode. -// It also returns the expected environment in a pod using -// the corresponding resource. -func (b *builder) flexibleParameters() ([]klog.KMetadata, []string) { - var objects []klog.KMetadata - switch b.driver.parameterMode { - case parameterModeConfigMap: - objects = append(objects, - b.parameters("x", "y"), - b.parameters("a", "b", "request_foo", "bar"), - ) - case parameterModeTranslated: - objects = append(objects, - b.parameters("x", "y"), - b.classParameters(b.parametersName(), "x", "y"), - b.parameters("a", "b", "request_foo", "bar"), - b.claimParameters(b.parametersName(), []string{"a", "b"}, []string{"request_foo", "bar"}), - ) - // The parameters object is not the last one but the second-last. - b.parametersCounter-- - case parameterModeStructured: - objects = append(objects, - b.classParameters("", "x", "y"), - b.claimParameters("", []string{"a", "b"}, []string{"request_foo", "bar"}), - ) - } - env := []string{"user_a", "b", "user_request_foo", "bar"} - if b.classParametersName != "" { - env = append(env, "admin_x", "y") - } - return objects, env -} - -// parametersName returns the current ConfigMap name for resource -// claim or class parameters. -func (b *builder) parametersName() string { - return fmt.Sprintf("parameters%s-%d", b.driver.NameSuffix, b.parametersCounter) -} - -// parametersEnv returns the default env variables. -func (b *builder) parametersEnv() map[string]string { - return map[string]string{ - "a": "b", - "request_foo": "bar", - } -} - -// parameters returns a config map with the default env variables. -func (b *builder) parameters(kv ...string) *v1.ConfigMap { - data := b.parameterData(kv...) - b.parametersCounter++ - return &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.f.Namespace.Name, - Name: b.parametersName(), - }, - Data: data, - } -} - -func (b *builder) classParameters(generatedFrom string, kv ...string) *resourceapi.ResourceClassParameters { - raw := b.rawParameterData(kv...) - b.parametersCounter++ - parameters := &resourceapi.ResourceClassParameters{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.f.Namespace.Name, - Name: b.parametersName(), - }, - - VendorParameters: []resourceapi.VendorParameters{ - {DriverName: b.driver.Name, Parameters: runtime.RawExtension{Raw: raw}}, - }, - } - - if generatedFrom != "" { - parameters.GeneratedFrom = &resourceapi.ResourceClassParametersReference{ - Kind: "ConfigMap", - Namespace: b.f.Namespace.Name, - Name: generatedFrom, - } - } - - return parameters -} - -func (b *builder) claimParameters(generatedFrom string, claimKV, requestKV []string) *resourceapi.ResourceClaimParameters { - b.parametersCounter++ - parameters := &resourceapi.ResourceClaimParameters{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.f.Namespace.Name, - Name: b.parametersName(), - }, - - // Without any request, nothing gets allocated and vendor - // parameters are also not passed down because they get - // attached to the allocation result. - DriverRequests: []resourceapi.DriverRequests{ - { - DriverName: b.driver.Name, - VendorParameters: runtime.RawExtension{Raw: b.rawParameterData(claimKV...)}, - Requests: []resourceapi.ResourceRequest{ - { - VendorParameters: runtime.RawExtension{Raw: b.rawParameterData(requestKV...)}, - ResourceRequestModel: resourceapi.ResourceRequestModel{ - NamedResources: &resourceapi.NamedResourcesRequest{ - Selector: "true", - }, +// claimSpec returns the device request for a claim or claim template +// with the associated config +func (b *builder) claimSpec() resourceapi.ResourceClaimSpec { + parameters, _ := b.parametersEnv() + spec := resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{{ + Name: "my-request", + DeviceClassName: b.className(), + }}, + Config: []resourceapi.DeviceClaimConfiguration{{ + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: b.driver.Name, + Parameters: runtime.RawExtension{ + Raw: []byte(parameters), }, }, }, - }, + }}, }, } - if generatedFrom != "" { - parameters.GeneratedFrom = &resourceapi.ResourceClaimParametersReference{ - Kind: "ConfigMap", - Name: generatedFrom, - } + if b.driver.parameterMode == parameterModeClassicDRA { + spec.Controller = b.driver.Name } - return parameters + return spec } -func (b *builder) parameterData(kv ...string) map[string]string { - data := map[string]string{} - for i := 0; i < len(kv); i += 2 { - data[kv[i]] = kv[i+1] - } - if len(data) == 0 { - data = b.parametersEnv() - } - return data -} - -func (b *builder) rawParameterData(kv ...string) []byte { - data := b.parameterData(kv...) - raw, err := json.Marshal(data) - framework.ExpectNoError(err, "JSON encoding of parameter data") - return raw +// parametersEnv returns the default user env variables as JSON (config) and key/value list (pod env). +func (b *builder) parametersEnv() (string, []string) { + return `{"a":"b"}`, + []string{"user_a", "b"} } // makePod returns a simple pod with no resource claims. @@ -1340,14 +1470,7 @@ func (b *builder) podInline() (*v1.Pod, *resourceapi.ResourceClaimTemplate) { Namespace: pod.Namespace, }, Spec: resourceapi.ResourceClaimTemplateSpec{ - Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: b.className(), - ParametersRef: &resourceapi.ResourceClaimParametersReference{ - APIGroup: b.driver.parameterAPIGroup, - Kind: b.driver.claimParameterAPIKind, - Name: b.parametersName(), - }, - }, + Spec: b.claimSpec(), }, } return pod, template @@ -1395,11 +1518,11 @@ func (b *builder) create(ctx context.Context, objs ...klog.KMetadata) []klog.KMe var err error var createdObj klog.KMetadata switch obj := obj.(type) { - case *resourceapi.ResourceClass: - createdObj, err = b.f.ClientSet.ResourceV1alpha3().ResourceClasses().Create(ctx, obj, metav1.CreateOptions{}) + case *resourceapi.DeviceClass: + createdObj, err = b.f.ClientSet.ResourceV1alpha3().DeviceClasses().Create(ctx, obj, metav1.CreateOptions{}) ginkgo.DeferCleanup(func(ctx context.Context) { - err := b.f.ClientSet.ResourceV1alpha3().ResourceClasses().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{}) - framework.ExpectNoError(err, "delete resource class") + err := b.f.ClientSet.ResourceV1alpha3().DeviceClasses().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err, "delete device class") }) case *v1.Pod: createdObj, err = b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) @@ -1409,10 +1532,6 @@ func (b *builder) create(ctx context.Context, objs ...klog.KMetadata) []klog.KMe createdObj, err = b.f.ClientSet.ResourceV1alpha3().ResourceClaims(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) case *resourceapi.ResourceClaimTemplate: createdObj, err = b.f.ClientSet.ResourceV1alpha3().ResourceClaimTemplates(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) - case *resourceapi.ResourceClassParameters: - createdObj, err = b.f.ClientSet.ResourceV1alpha3().ResourceClassParameters(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) - case *resourceapi.ResourceClaimParameters: - createdObj, err = b.f.ClientSet.ResourceV1alpha3().ResourceClaimParameters(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) case *resourceapi.ResourceSlice: createdObj, err = b.f.ClientSet.ResourceV1alpha3().ResourceSlices().Create(ctx, obj, metav1.CreateOptions{}) ginkgo.DeferCleanup(func(ctx context.Context) { @@ -1430,23 +1549,44 @@ func (b *builder) create(ctx context.Context, objs ...klog.KMetadata) []klog.KMe // testPod runs pod and checks if container logs contain expected environment variables func (b *builder) testPod(ctx context.Context, clientSet kubernetes.Interface, pod *v1.Pod, env ...string) { + ginkgo.GinkgoHelper() err := e2epod.WaitForPodRunningInNamespace(ctx, clientSet, pod) framework.ExpectNoError(err, "start pod") + if len(env) == 0 { + _, env = b.parametersEnv() + } for _, container := range pod.Spec.Containers { - log, err := e2epod.GetPodLogs(ctx, clientSet, pod.Namespace, pod.Name, container.Name) - framework.ExpectNoError(err, "get logs") - if len(env) == 0 { - for key, value := range b.parametersEnv() { - envStr := fmt.Sprintf("\nuser_%s=%s\n", key, value) - gomega.Expect(log).To(gomega.ContainSubstring(envStr), "container env variables") - } - } else { - for i := 0; i < len(env); i += 2 { - envStr := fmt.Sprintf("\n%s=%s\n", env[i], env[i+1]) - gomega.Expect(log).To(gomega.ContainSubstring(envStr), "container env variables") + testContainerEnv(ctx, clientSet, pod, container.Name, false, env...) + } +} + +// envLineRE matches env output with variables set by test/e2e/dra/test-driver. +var envLineRE = regexp.MustCompile(`^(?:admin|user|claim)_[a-zA-Z0-9_]*=.*$`) + +func testContainerEnv(ctx context.Context, clientSet kubernetes.Interface, pod *v1.Pod, containerName string, fullMatch bool, env ...string) { + ginkgo.GinkgoHelper() + log, err := e2epod.GetPodLogs(ctx, clientSet, pod.Namespace, pod.Name, containerName) + framework.ExpectNoError(err, fmt.Sprintf("get logs for container %s", containerName)) + if fullMatch { + // Find all env variables set by the test driver. + var actualEnv, expectEnv []string + for _, line := range strings.Split(log, "\n") { + if envLineRE.MatchString(line) { + actualEnv = append(actualEnv, line) } } + for i := 0; i < len(env); i += 2 { + expectEnv = append(expectEnv, env[i]+"="+env[i+1]) + } + sort.Strings(actualEnv) + sort.Strings(expectEnv) + gomega.Expect(actualEnv).To(gomega.Equal(expectEnv), fmt.Sprintf("container %s log output:\n%s", containerName, log)) + } else { + for i := 0; i < len(env); i += 2 { + envStr := fmt.Sprintf("\n%s=%s\n", env[i], env[i+1]) + gomega.Expect(log).To(gomega.ContainSubstring(envStr), fmt.Sprintf("container %s env variables", containerName)) + } } } @@ -1460,7 +1600,6 @@ func newBuilder(f *framework.Framework, driver *Driver) *builder { func (b *builder) setUp() { b.podCounter = 0 - b.parametersCounter = 0 b.claimCounter = 0 b.create(context.Background(), b.class()) ginkgo.DeferCleanup(b.tearDown) diff --git a/test/e2e/dra/kind.yaml b/test/e2e/dra/kind.yaml index 952f874812b..80146fe1fc7 100644 --- a/test/e2e/dra/kind.yaml +++ b/test/e2e/dra/kind.yaml @@ -14,6 +14,7 @@ nodes: scheduler: extraArgs: v: "5" + vmodule: "allocator=6,dynamicresources=6" # structured/allocator.go, DRA scheduler plugin controllerManager: extraArgs: v: "5" diff --git a/test/e2e/dra/test-driver/app/controller.go b/test/e2e/dra/test-driver/app/controller.go index 5a578ba3664..dc276b8cea4 100644 --- a/test/e2e/dra/test-driver/app/controller.go +++ b/test/e2e/dra/test-driver/app/controller.go @@ -20,16 +20,13 @@ package app import ( "context" - "encoding/json" "errors" "fmt" - "math/rand" - "strings" + "slices" "sync" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1alpha3" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/informers" @@ -48,7 +45,9 @@ type Resources struct { Nodes []string // NodeLabels are labels which determine on which nodes resources are // available. Mutually exclusive with Nodes. - NodeLabels labels.Set + NodeLabels labels.Set + + // Number of devices called "device-000", "device-001", ... on each node or in the cluster. MaxAllocations int // AllocateWrapper, if set, gets called for each Allocate call. @@ -68,12 +67,16 @@ func (r Resources) AllNodes(nodeLister listersv1.NodeLister) []string { return r.Nodes } -func (r Resources) NewAllocation(node string, data []byte) *resourceapi.AllocationResult { - allocation := &resourceapi.AllocationResult{} - allocation.ResourceHandles = []resourceapi.ResourceHandle{ - { - DriverName: r.DriverName, - Data: string(data), +func (r Resources) newAllocation(requestName, node string, config []resourceapi.DeviceAllocationConfiguration) *resourceapi.AllocationResult { + allocation := &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{{ + Driver: r.DriverName, + Pool: "none", + Request: requestName, + Device: "none", + }}, + Config: config, }, } if node == "" && len(r.NodeLabels) > 0 { @@ -86,7 +89,7 @@ func (r Resources) NewAllocation(node string, data []byte) *resourceapi.Allocati Values: []string{value}, }) } - allocation.AvailableOnNodes = &v1.NodeSelector{ + allocation.NodeSelector = &v1.NodeSelector{ NodeSelectorTerms: []v1.NodeSelectorTerm{ { MatchExpressions: requirements, @@ -103,7 +106,7 @@ func (r Resources) NewAllocation(node string, data []byte) *resourceapi.Allocati nodes = r.Nodes } if len(nodes) > 0 { - allocation.AvailableOnNodes = &v1.NodeSelector{ + allocation.NodeSelector = &v1.NodeSelector{ NodeSelectorTerms: []v1.NodeSelectorTerm{ { MatchExpressions: []v1.NodeSelectorRequirement{ @@ -166,11 +169,6 @@ func (c *ExampleController) Run(ctx context.Context, workers int) { informerFactory.Shutdown() } -type parameters struct { - EnvVars map[string]string - NodeName string -} - var _ controller.Driver = &ExampleController{} // GetNumAllocations returns the number of times that a claim was allocated. @@ -193,36 +191,6 @@ func (c *ExampleController) GetNumDeallocations() int64 { return c.numDeallocations } -func (c *ExampleController) GetClassParameters(ctx context.Context, class *resourceapi.ResourceClass) (interface{}, error) { - if class.ParametersRef != nil { - if class.ParametersRef.APIGroup != "" || - class.ParametersRef.Kind != "ConfigMap" { - return nil, fmt.Errorf("class parameters are only supported in APIVersion v1, Kind ConfigMap, got: %v", class.ParametersRef) - } - return c.readParametersFromConfigMap(ctx, class.ParametersRef.Namespace, class.ParametersRef.Name) - } - return nil, nil -} - -func (c *ExampleController) GetClaimParameters(ctx context.Context, claim *resourceapi.ResourceClaim, class *resourceapi.ResourceClass, classParameters interface{}) (interface{}, error) { - if claim.Spec.ParametersRef != nil { - if claim.Spec.ParametersRef.APIGroup != "" || - claim.Spec.ParametersRef.Kind != "ConfigMap" { - return nil, fmt.Errorf("claim parameters are only supported in APIVersion v1, Kind ConfigMap, got: %v", claim.Spec.ParametersRef) - } - return c.readParametersFromConfigMap(ctx, claim.Namespace, claim.Spec.ParametersRef.Name) - } - return nil, nil -} - -func (c *ExampleController) readParametersFromConfigMap(ctx context.Context, namespace, name string) (map[string]string, error) { - configMap, err := c.clientset.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("get config map: %w", err) - } - return configMap.Data, nil -} - func (c *ExampleController) Allocate(ctx context.Context, claimAllocations []*controller.ClaimAllocation, selectedNode string) { if c.resources.AllocateWrapper != nil { @@ -236,7 +204,7 @@ func (c *ExampleController) Allocate(ctx context.Context, claimAllocations []*co func (c *ExampleController) allocateOneByOne(ctx context.Context, claimAllocations []*controller.ClaimAllocation, selectedNode string) { for _, ca := range claimAllocations { - allocationResult, err := c.allocateOne(ctx, ca.Claim, ca.ClaimParameters, ca.Class, ca.ClassParameters, selectedNode) + allocationResult, err := c.allocateOne(ctx, ca.Claim, ca.DeviceClasses, selectedNode) if err != nil { ca.Error = err continue @@ -246,12 +214,25 @@ func (c *ExampleController) allocateOneByOne(ctx context.Context, claimAllocatio } // allocate simply copies parameters as JSON map into a ResourceHandle. -func (c *ExampleController) allocateOne(ctx context.Context, claim *resourceapi.ResourceClaim, claimParameters interface{}, class *resourceapi.ResourceClass, classParameters interface{}, selectedNode string) (result *resourceapi.AllocationResult, err error) { +func (c *ExampleController) allocateOne(ctx context.Context, claim *resourceapi.ResourceClaim, deviceClasses map[string]*resourceapi.DeviceClass, selectedNode string) (result *resourceapi.AllocationResult, err error) { logger := klog.LoggerWithValues(klog.LoggerWithName(klog.FromContext(ctx), "Allocate"), "claim", klog.KObj(claim), "uid", claim.UID) defer func() { logger.V(3).Info("done", "result", result, "err", err) }() + if len(claim.Spec.Devices.Requests) != 1 || + claim.Spec.Devices.Requests[0].DeviceClassName == "" || + claim.Spec.Devices.Requests[0].AllocationMode != resourceapi.DeviceAllocationModeExactCount || + claim.Spec.Devices.Requests[0].Count != 1 { + return nil, errors.New("only claims requesting exactly one device are supported") + } + request := claim.Spec.Devices.Requests[0] + class := deviceClasses[request.DeviceClassName] + if len(request.Selectors) > 0 || + class != nil && len(class.Spec.Selectors) > 0 { + return nil, errors.New("device selectors are not supported") + } + c.mutex.Lock() defer c.mutex.Unlock() @@ -267,24 +248,7 @@ func (c *ExampleController) allocateOne(ctx context.Context, claim *resourceapi. nodes := c.resources.AllNodes(c.nodeLister) if c.resources.NodeLocal { node = selectedNode - if node == "" { - // If none has been selected because we do immediate allocation, - // then we need to pick one ourselves. - var viableNodes []string - for _, n := range nodes { - if c.resources.MaxAllocations == 0 || - c.claimsPerNode[n] < c.resources.MaxAllocations { - viableNodes = append(viableNodes, n) - } - } - if len(viableNodes) == 0 { - return nil, errors.New("resources exhausted on all nodes") - } - // Pick randomly. We could also prefer the one with the least - // number of allocations (even spreading) or the most (packing). - node = viableNodes[rand.Intn(len(viableNodes))] - logger.V(3).Info("picked a node ourselves", "selectedNode", selectedNode) - } else if !contains(nodes, node) || + if !slices.Contains(nodes, node) || c.resources.MaxAllocations > 0 && c.claimsPerNode[node] >= c.resources.MaxAllocations { return nil, fmt.Errorf("resources exhausted on node %q", node) @@ -297,17 +261,47 @@ func (c *ExampleController) allocateOne(ctx context.Context, claim *resourceapi. } } - p := parameters{ - EnvVars: make(map[string]string), - NodeName: node, + var configs []resourceapi.DeviceAllocationConfiguration + for i, config := range claim.Spec.Devices.Config { + if len(config.Requests) != 0 && + !slices.Contains(config.Requests, request.Name) { + // Does not apply to request. + continue + } + if config.Opaque == nil { + return nil, fmt.Errorf("claim config #%d: only opaque configuration supported", i) + } + if config.Opaque.Driver != c.resources.DriverName { + // Does not apply to driver. + continue + } + // A normal driver would validate the config here. The test + // driver just passes it through. + configs = append(configs, + resourceapi.DeviceAllocationConfiguration{ + Source: resourceapi.AllocationConfigSourceClaim, + DeviceConfiguration: config.DeviceConfiguration, + }, + ) } - toEnvVars("user", claimParameters, p.EnvVars) - toEnvVars("admin", classParameters, p.EnvVars) - data, err := json.Marshal(p) - if err != nil { - return nil, fmt.Errorf("encode parameters: %w", err) + if class != nil { + for i, config := range class.Spec.Config { + if config.Opaque == nil { + return nil, fmt.Errorf("class config #%d: only opaque configuration supported", i) + } + if config.Opaque.Driver != c.resources.DriverName { + // Does not apply to driver. + continue + } + configs = append(configs, + resourceapi.DeviceAllocationConfiguration{ + Source: resourceapi.AllocationConfigSourceClass, + DeviceConfiguration: config.DeviceConfiguration, + }, + ) + } } - allocation := c.resources.NewAllocation(node, data) + allocation := c.resources.newAllocation(request.Name, node, configs) if !alreadyAllocated { c.numAllocations++ c.allocated[claim.UID] = node @@ -359,7 +353,7 @@ func (c *ExampleController) UnsuitableNodes(ctx context.Context, pod *v1.Pod, cl // can only work if a node has capacity left // for all of them. Also, nodes that the driver // doesn't run on cannot be used. - if !contains(nodes, node) || + if !slices.Contains(nodes, node) || c.claimsPerNode[node]+len(claims) > c.resources.MaxAllocations { claim.UnsuitableNodes = append(claim.UnsuitableNodes, node) } @@ -372,7 +366,7 @@ func (c *ExampleController) UnsuitableNodes(ctx context.Context, pod *v1.Pod, cl for _, claim := range claims { claim.UnsuitableNodes = nil for _, node := range potentialNodes { - if !contains(nodes, node) || + if !slices.Contains(nodes, node) || allocations+len(claims) > c.resources.MaxAllocations { claim.UnsuitableNodes = append(claim.UnsuitableNodes, node) } @@ -381,24 +375,3 @@ func (c *ExampleController) UnsuitableNodes(ctx context.Context, pod *v1.Pod, cl return nil } - -func toEnvVars(what string, from interface{}, to map[string]string) { - if from == nil { - return - } - - env := from.(map[string]string) - for key, value := range env { - to[what+"_"+strings.ToLower(key)] = value - } -} - -func contains[T comparable](list []T, value T) bool { - for _, v := range list { - if v == value { - return true - } - } - - return false -} diff --git a/test/e2e/dra/test-driver/app/kubeletplugin.go b/test/e2e/dra/test-driver/app/kubeletplugin.go index 5b7c1f07422..da6d1f555b3 100644 --- a/test/e2e/dra/test-driver/app/kubeletplugin.go +++ b/test/e2e/dra/test-driver/app/kubeletplugin.go @@ -23,6 +23,9 @@ import ( "fmt" "os" "path/filepath" + "regexp" + "slices" + "sort" "strings" "sync" @@ -46,15 +49,14 @@ type ExamplePlugin struct { d kubeletplugin.DRAPlugin fileOps FileOperations - cdiDir string - driverName string - nodeName string - instances sets.Set[string] + cdiDir string + driverName string + nodeName string + deviceNames sets.Set[string] - mutex sync.Mutex - instancesInUse sets.Set[string] - prepared map[ClaimID][]string // instance names - gRPCCalls []GRPCCall + mutex sync.Mutex + prepared map[ClaimID][]Device // prepared claims -> result of nodePrepareResource + gRPCCalls []GRPCCall blockPrepareResourcesMutex sync.Mutex blockUnprepareResourcesMutex sync.Mutex @@ -88,11 +90,18 @@ type ClaimID struct { UID string } +type Device struct { + PoolName string + DeviceName string + RequestName string + CDIDeviceID string +} + var _ drapb.NodeServer = &ExamplePlugin{} // getJSONFilePath returns the absolute path where CDI file is/should be. -func (ex *ExamplePlugin) getJSONFilePath(claimUID string) string { - return filepath.Join(ex.cdiDir, fmt.Sprintf("%s-%s.json", ex.driverName, claimUID)) +func (ex *ExamplePlugin) getJSONFilePath(claimUID string, requestName string) string { + return filepath.Join(ex.cdiDir, fmt.Sprintf("%s-%s-%s.json", ex.driverName, claimUID, requestName)) } // FileOperations defines optional callbacks for handling CDI files @@ -105,10 +114,13 @@ type FileOperations struct { // file does not exist. Remove func(name string) error - // NumResourceInstances determines whether the plugin reports resources - // instances and how many. A negative value causes it to report "not implemented" - // in the NodeListAndWatchResources gRPC call. - NumResourceInstances int + // NumDevices determines whether the plugin reports devices + // and how many. It reports nothing if negative. + NumDevices int + + // Pre-defined devices, with each device name mapped to + // the device attributes. Not used if NumDevices >= 0. + Devices map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute } // StartPlugin sets up the servers that are necessary for a DRA kubelet plugin. @@ -129,22 +141,23 @@ func StartPlugin(ctx context.Context, cdiDir, driverName string, kubeClient kube } } ex := &ExamplePlugin{ - stopCh: ctx.Done(), - logger: logger, - kubeClient: kubeClient, - fileOps: fileOps, - cdiDir: cdiDir, - driverName: driverName, - nodeName: nodeName, - instances: sets.New[string](), - instancesInUse: sets.New[string](), - prepared: make(map[ClaimID][]string), + stopCh: ctx.Done(), + logger: logger, + kubeClient: kubeClient, + fileOps: fileOps, + cdiDir: cdiDir, + driverName: driverName, + nodeName: nodeName, + prepared: make(map[ClaimID][]Device), + deviceNames: sets.New[string](), } - for i := 0; i < ex.fileOps.NumResourceInstances; i++ { - ex.instances.Insert(fmt.Sprintf("instance-%02d", i)) + for i := 0; i < ex.fileOps.NumDevices; i++ { + ex.deviceNames.Insert(fmt.Sprintf("device-%02d", i)) + } + for deviceName := range ex.fileOps.Devices { + ex.deviceNames.Insert(deviceName) } - opts = append(opts, kubeletplugin.DriverName(driverName), kubeletplugin.NodeName(nodeName), @@ -158,19 +171,30 @@ func StartPlugin(ctx context.Context, cdiDir, driverName string, kubeClient kube } ex.d = d - if fileOps.NumResourceInstances >= 0 { - instances := make([]resourceapi.NamedResourcesInstance, ex.fileOps.NumResourceInstances) - for i := 0; i < ex.fileOps.NumResourceInstances; i++ { - instances[i].Name = fmt.Sprintf("instance-%02d", i) + if fileOps.NumDevices >= 0 { + devices := make([]resourceapi.Device, ex.fileOps.NumDevices) + for i := 0; i < ex.fileOps.NumDevices; i++ { + devices[i] = resourceapi.Device{ + Name: fmt.Sprintf("device-%02d", i), + Basic: &resourceapi.BasicDevice{}, + } } - nodeResources := []*resourceapi.ResourceModel{ - { - NamedResources: &resourceapi.NamedResourcesResources{ - Instances: instances, - }, - }, + resources := kubeletplugin.Resources{ + Devices: devices, } - ex.d.PublishResources(ctx, nodeResources) + ex.d.PublishResources(ctx, resources) + } else if len(ex.fileOps.Devices) > 0 { + devices := make([]resourceapi.Device, len(ex.fileOps.Devices)) + for i, deviceName := range sets.List(ex.deviceNames) { + devices[i] = resourceapi.Device{ + Name: deviceName, + Basic: &resourceapi.BasicDevice{Attributes: ex.fileOps.Devices[deviceName]}, + } + } + resources := kubeletplugin.Resources{ + Devices: devices, + } + ex.d.PublishResources(ctx, resources) } return ex, nil @@ -245,17 +269,15 @@ func (ex *ExamplePlugin) getUnprepareResourcesFailure() error { return ex.unprepareResourcesFailure } -// NodePrepareResource ensures that the CDI file for the claim exists. It uses +// NodePrepareResource ensures that the CDI file(s) (one per request) for the claim exists. It uses // a deterministic name to simplify NodeUnprepareResource (no need to remember // or discover the name) and idempotency (when called again, the file simply // gets written again). -func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drapb.Claim) ([]string, error) { +func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drapb.Claim) ([]Device, error) { logger := klog.FromContext(ctx) // The plugin must retrieve the claim itself to get it in the version // that it understands. - var resourceHandle string - var structuredResourceHandle *resourceapi.StructuredResourceHandle claim, err := ex.kubeClient.ResourceV1alpha3().ResourceClaims(claimReq.Namespace).Get(ctx, claimReq.Name, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("retrieve claim %s/%s: %w", claimReq.Namespace, claimReq.Name, err) @@ -263,127 +285,113 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drap if claim.Status.Allocation == nil { return nil, fmt.Errorf("claim %s/%s not allocated", claimReq.Namespace, claimReq.Name) } - if claim.UID != types.UID(claimReq.Uid) { + if claim.UID != types.UID(claimReq.UID) { return nil, fmt.Errorf("claim %s/%s got replaced", claimReq.Namespace, claimReq.Name) } - haveResources := false - for _, handle := range claim.Status.Allocation.ResourceHandles { - if handle.DriverName == ex.driverName { - haveResources = true - resourceHandle = handle.Data - structuredResourceHandle = handle.StructuredData - break - } - } - if !haveResources { - // Nothing to do. - return nil, nil - } ex.mutex.Lock() defer ex.mutex.Unlock() ex.blockPrepareResourcesMutex.Lock() defer ex.blockPrepareResourcesMutex.Unlock() - deviceName := "claim-" + claimReq.Uid - vendor := ex.driverName - class := "test" - dev := vendor + "/" + class + "=" + deviceName - claimID := ClaimID{Name: claimReq.Name, UID: claimReq.Uid} - if _, ok := ex.prepared[claimID]; ok { + claimID := ClaimID{Name: claimReq.Name, UID: claimReq.UID} + if result, ok := ex.prepared[claimID]; ok { // Idempotent call, nothing to do. - return []string{dev}, nil + return result, nil } - // Determine environment variables. - var p parameters - var instanceNames []string - if structuredResourceHandle == nil { - // Control plane controller did the allocation. - if err := json.Unmarshal([]byte(resourceHandle), &p); err != nil { - return nil, fmt.Errorf("unmarshal resource handle: %w", err) - } - } else { - // Scheduler did the allocation with structured parameters. - p.NodeName = structuredResourceHandle.NodeName - if err := extractParameters(structuredResourceHandle.VendorClassParameters, &p.EnvVars, "admin"); err != nil { - return nil, err - } - if err := extractParameters(structuredResourceHandle.VendorClaimParameters, &p.EnvVars, "user"); err != nil { - return nil, err - } - for _, result := range structuredResourceHandle.Results { - if err := extractParameters(result.VendorRequestParameters, &p.EnvVars, "user"); err != nil { - return nil, err - } - namedResources := result.NamedResources - if namedResources == nil { - return nil, errors.New("missing named resources allocation result") - } - instanceName := namedResources.Name - if instanceName == "" { - return nil, errors.New("empty named resources instance name") - } - if !ex.instances.Has(instanceName) { - return nil, fmt.Errorf("unknown allocated instance %q", instanceName) - } - if ex.instancesInUse.Has(instanceName) { - return nil, fmt.Errorf("resource instance %q used more than once", instanceName) - } - instanceNames = append(instanceNames, instanceName) - } - } + var devices []Device + for _, result := range claim.Status.Allocation.Devices.Results { + requestName := result.Request - // Sanity check scheduling. - if p.NodeName != "" && ex.nodeName != "" && p.NodeName != ex.nodeName { - return nil, fmt.Errorf("claim was allocated for %q, cannot be prepared on %q", p.NodeName, ex.nodeName) - } + // The driver joins all env variables in the order in which + // they appear in results (last one wins). + env := make(map[string]string) + for i, config := range claim.Status.Allocation.Devices.Config { + if config.Opaque == nil || + config.Opaque.Driver != ex.driverName || + len(config.Requests) > 0 && !slices.Contains(config.Requests, requestName) { + continue + } + if err := extractParameters(config.Opaque.Parameters, &env, config.Source == resourceapi.AllocationConfigSourceClass); err != nil { + return nil, fmt.Errorf("parameters in config #%d: %w", i, err) + } + } - // CDI wants env variables as set of strings. - envs := []string{} - for key, val := range p.EnvVars { - envs = append(envs, key+"="+val) - } + // It also sets a claim__=true env variable. + // This can be used to identify which devices where mapped into a container. + claimReqName := "claim_" + claim.Name + "_" + requestName + claimReqName = regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(claimReqName, "_") + env[claimReqName] = "true" - spec := &spec{ - Version: "0.3.0", // This has to be a version accepted by the runtimes. - Kind: vendor + "/" + class, - // At least one device is required and its entry must have more - // than just the name. - Devices: []device{ - { - Name: deviceName, - ContainerEdits: containerEdits{ - Env: envs, + deviceName := "claim-" + claimReq.UID + "-" + requestName + vendor := ex.driverName + class := "test" + cdiDeviceID := vendor + "/" + class + "=" + deviceName + + // CDI wants env variables as set of strings. + envs := []string{} + for key, val := range env { + envs = append(envs, key+"="+val) + } + sort.Strings(envs) + + if len(envs) == 0 { + // CDI does not support empty ContainerEdits. For example, + // kubelet+crio then fail with: + // CDI device injection failed: unresolvable CDI devices ... + // + // Inject nothing instead, which is supported by DRA. + continue + } + + spec := &spec{ + Version: "0.3.0", // This has to be a version accepted by the runtimes. + Kind: vendor + "/" + class, + // At least one device is required and its entry must have more + // than just the name. + Devices: []device{ + { + Name: deviceName, + ContainerEdits: containerEdits{ + Env: envs, + }, }, }, - }, - } - filePath := ex.getJSONFilePath(claimReq.Uid) - buffer, err := json.Marshal(spec) - if err != nil { - return nil, fmt.Errorf("marshal spec: %w", err) - } - if err := ex.fileOps.Create(filePath, buffer); err != nil { - return nil, fmt.Errorf("failed to write CDI file %v", err) + } + filePath := ex.getJSONFilePath(claimReq.UID, requestName) + buffer, err := json.Marshal(spec) + if err != nil { + return nil, fmt.Errorf("marshal spec: %w", err) + } + if err := ex.fileOps.Create(filePath, buffer); err != nil { + return nil, fmt.Errorf("failed to write CDI file: %w", err) + } + device := Device{ + PoolName: result.Pool, + DeviceName: result.Device, + RequestName: requestName, + CDIDeviceID: cdiDeviceID, + } + devices = append(devices, device) } - ex.prepared[claimID] = instanceNames - for _, instanceName := range instanceNames { - ex.instancesInUse.Insert(instanceName) - } - - logger.V(3).Info("CDI file created", "path", filePath, "device", dev) - return []string{dev}, nil + logger.V(3).Info("CDI file(s) created", "devices", devices) + ex.prepared[claimID] = devices + return devices, nil } -func extractParameters(parameters runtime.RawExtension, env *map[string]string, kind string) error { +func extractParameters(parameters runtime.RawExtension, env *map[string]string, admin bool) error { if len(parameters.Raw) == 0 { return nil } + kind := "user" + if admin { + kind = "admin" + } var data map[string]string if err := json.Unmarshal(parameters.Raw, &data); err != nil { - return fmt.Errorf("decoding %s parameters: %v", kind, err) + return fmt.Errorf("decoding %s parameters: %w", kind, err) } if len(data) > 0 && *env == nil { *env = make(map[string]string) @@ -404,15 +412,23 @@ func (ex *ExamplePlugin) NodePrepareResources(ctx context.Context, req *drapb.No } for _, claimReq := range req.Claims { - cdiDevices, err := ex.nodePrepareResource(ctx, claimReq) + devices, err := ex.nodePrepareResource(ctx, claimReq) if err != nil { - resp.Claims[claimReq.Uid] = &drapb.NodePrepareResourceResponse{ + resp.Claims[claimReq.UID] = &drapb.NodePrepareResourceResponse{ Error: err.Error(), } } else { - resp.Claims[claimReq.Uid] = &drapb.NodePrepareResourceResponse{ - CDIDevices: cdiDevices, + r := &drapb.NodePrepareResourceResponse{} + for _, device := range devices { + pbDevice := &drapb.Device{ + PoolName: device.PoolName, + DeviceName: device.DeviceName, + RequestNames: []string{device.RequestName}, + CDIDeviceIDs: []string{device.CDIDeviceID}, + } + r.Devices = append(r.Devices, pbDevice) } + resp.Claims[claimReq.UID] = r } } return resp, nil @@ -427,27 +443,23 @@ func (ex *ExamplePlugin) nodeUnprepareResource(ctx context.Context, claimReq *dr logger := klog.FromContext(ctx) - filePath := ex.getJSONFilePath(claimReq.Uid) - if err := ex.fileOps.Remove(filePath); err != nil { - return fmt.Errorf("error removing CDI file: %w", err) - } - logger.V(3).Info("CDI file removed", "path", filePath) - - ex.mutex.Lock() - defer ex.mutex.Unlock() - - claimID := ClaimID{Name: claimReq.Name, UID: claimReq.Uid} - instanceNames, ok := ex.prepared[claimID] + claimID := ClaimID{Name: claimReq.Name, UID: claimReq.UID} + devices, ok := ex.prepared[claimID] if !ok { // Idempotent call, nothing to do. return nil } - delete(ex.prepared, claimID) - for _, instanceName := range instanceNames { - ex.instancesInUse.Delete(instanceName) + for _, device := range devices { + filePath := ex.getJSONFilePath(claimReq.UID, device.RequestName) + if err := ex.fileOps.Remove(filePath); err != nil { + return fmt.Errorf("error removing CDI file: %w", err) + } + logger.V(3).Info("CDI file removed", "path", filePath) } + delete(ex.prepared, claimID) + return nil } @@ -463,11 +475,11 @@ func (ex *ExamplePlugin) NodeUnprepareResources(ctx context.Context, req *drapb. for _, claimReq := range req.Claims { err := ex.nodeUnprepareResource(ctx, claimReq) if err != nil { - resp.Claims[claimReq.Uid] = &drapb.NodeUnprepareResourceResponse{ + resp.Claims[claimReq.UID] = &drapb.NodeUnprepareResourceResponse{ Error: err.Error(), } } else { - resp.Claims[claimReq.Uid] = &drapb.NodeUnprepareResourceResponse{} + resp.Claims[claimReq.UID] = &drapb.NodeUnprepareResourceResponse{} } } return resp, nil diff --git a/test/e2e/dra/test-driver/deploy/example/deviceclass.yaml b/test/e2e/dra/test-driver/deploy/example/deviceclass.yaml new file mode 100644 index 00000000000..44f4fcf4686 --- /dev/null +++ b/test/e2e/dra/test-driver/deploy/example/deviceclass.yaml @@ -0,0 +1,8 @@ +apiVersion: resource.k8s.io/v1alpha3 +kind: ResourceClass +metadata: + name: example +spec: + selectors: + - cel: + expression: device.driver == "test-driver.cdi.k8s.io" diff --git a/test/e2e/dra/test-driver/deploy/example/plugin-permissions.yaml b/test/e2e/dra/test-driver/deploy/example/plugin-permissions.yaml index 80e12ee5e4c..d869339c22b 100644 --- a/test/e2e/dra/test-driver/deploy/example/plugin-permissions.yaml +++ b/test/e2e/dra/test-driver/deploy/example/plugin-permissions.yaml @@ -47,7 +47,7 @@ spec: matchConstraints: resourceRules: - apiGroups: ["resource.k8s.io"] - apiVersions: ["v1alpha2"] + apiVersions: ["v1alpha3"] operations: ["CREATE", "UPDATE", "DELETE"] resources: ["resourceslices"] variables: @@ -59,7 +59,7 @@ spec: request.userInfo.username == "system:serviceaccount:dra-kubelet-plugin-namespace:dra-kubelet-plugin-service-account" - name: objectNodeName expression: >- - (request.operation == "DELETE" ? oldObject : object).?nodeName.orValue("") + (request.operation == "DELETE" ? oldObject : object).spec.?nodeName.orValue("") validations: - expression: >- !variables.isKubeletPlugin || variables.hasNodeName diff --git a/test/e2e/dra/test-driver/deploy/example/resourceclaim.yaml b/test/e2e/dra/test-driver/deploy/example/resourceclaim.yaml index 7d2f774b55f..ec965e847e2 100644 --- a/test/e2e/dra/test-driver/deploy/example/resourceclaim.yaml +++ b/test/e2e/dra/test-driver/deploy/example/resourceclaim.yaml @@ -1,18 +1,10 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: example-claim-parameters - namespace: default -data: - a: b ---- apiVersion: resource.k8s.io/v1alpha3 kind: ResourceClaim metadata: name: example namespace: default spec: - resourceClassName: example - parametersRef: - kind: ConfigMap - name: example-claim-parameters + devices: + requests: + - name: req-0 + deviceClassName: example diff --git a/test/e2e/dra/test-driver/deploy/example/resourceclass.yaml b/test/e2e/dra/test-driver/deploy/example/resourceclass.yaml deleted file mode 100644 index 89b856295e3..00000000000 --- a/test/e2e/dra/test-driver/deploy/example/resourceclass.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: resource.k8s.io/v1alpha3 -kind: ResourceClass -metadata: - name: example -driverName: test-driver.cdi.k8s.io -# TODO: -# parameters diff --git a/test/e2e_node/dra_test.go b/test/e2e_node/dra_test.go index 65f30a013ae..fc6c53f4d50 100644 --- a/test/e2e_node/dra_test.go +++ b/test/e2e_node/dra_test.go @@ -30,17 +30,20 @@ import ( "os" "path" "path/filepath" + "regexp" + "sort" + "strings" "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" "github.com/onsi/gomega/types" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1alpha3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" draplugin "k8s.io/kubernetes/pkg/kubelet/cm/dra/plugin" @@ -417,10 +420,9 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, }) ginkgo.It("must run pod if NodePrepareResources is in progress for one plugin when Kubelet restarts", func(ctx context.Context) { - _, kubeletPlugin2 := start(ctx) - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin1, kubeletPlugin2 := start(ctx) - unblock := kubeletPlugin.BlockNodePrepareResources() + unblock := kubeletPlugin1.BlockNodePrepareResources() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{kubeletPlugin1Name, kubeletPlugin2Name}) ginkgo.By("wait for pod to be in Pending state") @@ -478,9 +480,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, } matchResourcesByNodeName := func(nodeName string) types.GomegaMatcher { - return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "NodeName": gomega.Equal(nodeName), - }) + return gomega.HaveField("Spec.NodeName", gomega.Equal(nodeName)) } f.It("must be removed on kubelet startup", f.WithDisruptive(), func(ctx context.Context) { @@ -562,7 +562,7 @@ func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeN ginkgo.DeferCleanup(func(ctx context.Context) { // kubelet should do this eventually, but better make sure. // A separate test checks this explicitly. - framework.ExpectNoError(clientSet.ResourceV1alpha3().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: "driverName=" + driverName})) + framework.ExpectNoError(clientSet.ResourceV1alpha3().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName})) }) ginkgo.DeferCleanup(plugin.Stop) @@ -573,18 +573,17 @@ func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeN // NOTE: as scheduler and controller manager are not running by the Node e2e, // the objects must contain all required data to be processed correctly by the API server // and placed on the node without involving the scheduler and the DRA controller -func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, nodename, namespace, className, claimName, podName string, deferPodDeletion bool, pluginNames []string) *v1.Pod { - // ResourceClass - class := &resourceapi.ResourceClass{ +func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, nodename, namespace, className, claimName, podName string, deferPodDeletion bool, driverNames []string) *v1.Pod { + // DeviceClass + class := &resourceapi.DeviceClass{ ObjectMeta: metav1.ObjectMeta{ Name: className, }, - DriverName: "controller", } - _, err := clientSet.ResourceV1alpha3().ResourceClasses().Create(ctx, class, metav1.CreateOptions{}) + _, err := clientSet.ResourceV1alpha3().DeviceClasses().Create(ctx, class, metav1.CreateOptions{}) framework.ExpectNoError(err) - ginkgo.DeferCleanup(clientSet.ResourceV1alpha3().ResourceClasses().Delete, className, metav1.DeleteOptions{}) + ginkgo.DeferCleanup(clientSet.ResourceV1alpha3().DeviceClasses().Delete, className, metav1.DeleteOptions{}) // ResourceClaim podClaimName := "resource-claim" @@ -593,7 +592,12 @@ func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, node Name: claimName, }, Spec: resourceapi.ResourceClaimSpec{ - ResourceClassName: className, + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{{ + Name: "my-request", + DeviceClassName: className, + }}, + }, }, } createdClaim, err := clientSet.ResourceV1alpha3().ResourceClaims(namespace).Create(ctx, claim, metav1.CreateOptions{}) @@ -601,7 +605,18 @@ func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, node ginkgo.DeferCleanup(clientSet.ResourceV1alpha3().ResourceClaims(namespace).Delete, claimName, metav1.DeleteOptions{}) - // Pod + // The pod checks its own env with grep. Each driver injects its own parameters, + // with the driver name as part of the variable name. Sorting ensures that a + // single grep can match the output of env when that gets turned into a single + // line because the order is deterministic. + nameToEnv := func(driverName string) string { + return "DRA_" + regexp.MustCompile(`[^a-z0-9]`).ReplaceAllString(driverName, "_") + } + var expectedEnv []string + sort.Strings(driverNames) + for _, driverName := range driverNames { + expectedEnv = append(expectedEnv, nameToEnv(driverName)+"=PARAM1_VALUE") + } containerName := "testcontainer" pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -623,7 +638,9 @@ func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, node Resources: v1.ResourceRequirements{ Claims: []v1.ResourceClaim{{Name: podClaimName}}, }, - Command: []string{"/bin/sh", "-c", "env | grep DRA_PARAM1=PARAM1_VALUE"}, + // If injecting env variables fails, the pod fails and this error shows up in + // ... Terminated:&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:ERROR: ... + Command: []string{"/bin/sh", "-c", "if ! echo $(env) | grep -q " + strings.Join(expectedEnv, ".*") + "; then echo ERROR: unexpected env: $(env) >/dev/termination-log; exit 1 ; fi"}, }, }, RestartPolicy: v1.RestartPolicyNever, @@ -637,21 +654,36 @@ func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, node } // Update claim status: set ReservedFor and AllocationResult - // NOTE: This is usually done by the DRA controller - resourceHandlers := make([]resourceapi.ResourceHandle, len(pluginNames)) - for i, pluginName := range pluginNames { - resourceHandlers[i] = resourceapi.ResourceHandle{ - DriverName: pluginName, - Data: "{\"EnvVars\":{\"DRA_PARAM1\":\"PARAM1_VALUE\"},\"NodeName\":\"\"}", + // NOTE: This is usually done by the DRA controller or the scheduler. + results := make([]resourceapi.DeviceRequestAllocationResult, len(driverNames)) + config := make([]resourceapi.DeviceAllocationConfiguration, len(driverNames)) + for i, driverName := range driverNames { + results[i] = resourceapi.DeviceRequestAllocationResult{ + Driver: driverName, + Pool: "some-pool", + Device: "some-device", + Request: claim.Spec.Devices.Requests[0].Name, + } + config[i] = resourceapi.DeviceAllocationConfiguration{ + Source: resourceapi.AllocationConfigSourceClaim, + DeviceConfiguration: resourceapi.DeviceConfiguration{ + Opaque: &resourceapi.OpaqueDeviceConfiguration{ + Driver: driverName, + Parameters: runtime.RawExtension{Raw: []byte(`{"` + nameToEnv(driverName) + `":"PARAM1_VALUE"}`)}, + }, + }, } } + createdClaim.Status = resourceapi.ResourceClaimStatus{ - DriverName: "controller", ReservedFor: []resourceapi.ResourceClaimConsumerReference{ {Resource: "pods", Name: podName, UID: createdPod.UID}, }, Allocation: &resourceapi.AllocationResult{ - ResourceHandles: resourceHandlers, + Devices: resourceapi.DeviceAllocationResult{ + Results: results, + Config: config, + }, }, } _, err = clientSet.ResourceV1alpha3().ResourceClaims(namespace).UpdateStatus(ctx, createdClaim, metav1.UpdateOptions{}) @@ -665,10 +697,13 @@ func createTestResourceSlice(ctx context.Context, clientSet kubernetes.Interface ObjectMeta: metav1.ObjectMeta{ Name: nodeName, }, - NodeName: nodeName, - DriverName: driverName, - ResourceModel: resourceapi.ResourceModel{ - NamedResources: &resourceapi.NamedResourcesResources{}, + Spec: resourceapi.ResourceSliceSpec{ + NodeName: nodeName, + Driver: driverName, + Pool: resourceapi.ResourcePool{ + Name: nodeName, + ResourceSliceCount: 1, + }, }, }