From fb5099758cee91bc14ed8f17a20f1ad03183ac8c Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Sun, 22 Mar 2026 18:41:38 +0200 Subject: [PATCH 1/4] feat: DRA resource.k8s.io/v1 integration via draclient Migrate Dynamic Resource Allocation from kubelet PodResources / v1alpha2-style usage to the stable resource.k8s.io/v1 API (Kubernetes 1.34+). - Add pkg/draclient: fetch ResourceClaims and ResourceSlices, build pod resource map from device attributes (k8s.cni.cncf.io/deviceID, k8s.cni.cncf.io/resourceName) and ExtendedResourceClaimStatus - Wire GetPodResourceMap into k8sclient; remove DRA path from kubeletclient - RBAC: resourceclaims, resourceclaims/status, resourceslices (get, list) on multus ClusterRole - Docs: DRA / NAD usage; tests for draclient and k8sclient Co-authored-by: Sebastian Sch Signed-off-by: Fred Rolland --- deployments/multus-daemonset-crio.yml | 11 + deployments/multus-daemonset-thick.yml | 9 + deployments/multus-daemonset.yml | 11 + docs/how-to-use.md | 160 ++- pkg/draclient/draclient.go | 289 +++++ pkg/draclient/draclient_suite_test.go | 29 + pkg/draclient/draclient_test.go | 1552 +++++++++++++++++++++++ pkg/k8sclient/k8sclient.go | 14 +- pkg/k8sclient/k8sclient_test.go | 579 +++++++++ pkg/kubeletclient/kubeletclient.go | 24 - pkg/kubeletclient/kubeletclient_test.go | 32 +- pkg/logging/logging.go | 5 + pkg/multus/multus.go | 2 +- 13 files changed, 2598 insertions(+), 119 deletions(-) create mode 100644 pkg/draclient/draclient.go create mode 100644 pkg/draclient/draclient_suite_test.go create mode 100644 pkg/draclient/draclient_test.go diff --git a/deployments/multus-daemonset-crio.yml b/deployments/multus-daemonset-crio.yml index e3c28d4ff..bcd40ebe2 100644 --- a/deployments/multus-daemonset-crio.yml +++ b/deployments/multus-daemonset-crio.yml @@ -70,7 +70,18 @@ rules: - pods/status verbs: - get + - list - update + - watch + - apiGroups: + - "resource.k8s.io" + resources: + - resourceclaims + - resourceclaims/status + - resourceslices + verbs: + - get + - list - apiGroups: - "" - events.k8s.io diff --git a/deployments/multus-daemonset-thick.yml b/deployments/multus-daemonset-thick.yml index 1d9b12cc3..4840e869e 100644 --- a/deployments/multus-daemonset-thick.yml +++ b/deployments/multus-daemonset-thick.yml @@ -73,6 +73,15 @@ rules: - list - update - watch + - apiGroups: + - "resource.k8s.io" + resources: + - resourceclaims + - resourceclaims/status + - resourceslices + verbs: + - get + - list - apiGroups: - "" - events.k8s.io diff --git a/deployments/multus-daemonset.yml b/deployments/multus-daemonset.yml index 29cdcc4c7..dc8ae9cf7 100644 --- a/deployments/multus-daemonset.yml +++ b/deployments/multus-daemonset.yml @@ -70,7 +70,18 @@ rules: - pods/status verbs: - get + - list - update + - watch + - apiGroups: + - "resource.k8s.io" + resources: + - resourceclaims + - resourceclaims/status + - resourceslices + verbs: + - get + - list - apiGroups: - "" - events.k8s.io diff --git a/docs/how-to-use.md b/docs/how-to-use.md index b4cdfc426..58f1efa04 100644 --- a/docs/how-to-use.md +++ b/docs/how-to-use.md @@ -645,112 +645,139 @@ If you wish to have auto configuration use the `readinessindicatorfile` in the c ### Run pod with network annotation and Dynamic Resource Allocation driver -> :warning: Dynamic Resource Allocation (DRA) is [currently an alpha](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/), -> and is subject to change. Please consider this functionality as a preview. The architecture and usage of DRA in -> Multus CNI may change in the future as this technology matures. -> -> The current DRA integration is based on the DRA API for Kubernetes 1.26 to 1.30. With Kubernetes 1.31, the DRA API -> will change and multus doesn't integrate with the new API yet. -Dynamic Resource Allocation is alternative mechanism to device plugin which allows to requests pod and container -resources. +Dynamic Resource Allocation is an alternative mechanism to device plugin which allows pods to request pod and container +resources dynamically. -The following sections describe how to use DRA with multus and NVIDIA DRA driver. Other DRA networking driver vendors -should follow similar concepts to make use of multus DRA support. +The following sections describe how to use DRA with Multus. DRA networking driver vendors should follow similar +concepts to make use of Multus DRA support. #### Prerequisite -1. Kubernetes 1.27 -2. Container Runtime with CDI support enabled -3. Kubernetes runtime-config=resource.k8s.io/v1alpha2 -4. Kubernetes feature-gates=DynamicResourceAllocation=True,KubeletPodResourcesDynamicResources=true +1. Kubernetes 1.34+ #### Install DRA driver -The current example uses NVIDIA DRA driver for networking. This DRA driver is not publicly available. An alternative to -this DRA driver is available at [dra-example-driver](https://github.com/kubernetes-sigs/dra-example-driver). +You need to install a DRA driver that provides network devices. For example, you can use the SR-IOV DRA driver or +other DRA networking drivers. Refer to your DRA driver documentation for installation instructions. -#### Create dynamic resource class with NVIDIA network DRA driver +The DRA driver MUST expose the following attributes on each allocated **device** in `ResourceSlice`: +- `k8s.cni.cncf.io/deviceID`: device ID that Multus passes to the CNI plugin. +- `k8s.cni.cncf.io/resourceName`: **must exactly match** the value you put on the NetworkAttachmentDefinition + `k8s.v1.cni.cncf.io/resourceName` annotation (same style as classic extended resources, e.g. `intel.com/sriov_vf`). + If `k8s.cni.cncf.io/resourceName` is missing on the device, allocation processing fails. -The `ResourceClass` defines the resource pool of `sf-pool-1`. - -``` -# Execute following command at Kubernetes master -cat < 1", func() { + claimName := "multus-dual-port-extended-resources-ftknr" + device1Name := "0000-08-00-3" + device2Name := "0000-08-00-4" + device3Name := "0000-08-02-4" + driverName := "sriovnetwork.k8snetworkplumbingwg.io" + poolName := "c-237-169-100-104" + deviceID1 := "0000:08:00.3" + deviceID2 := "0000:08:00.4" + deviceID3 := "0000:08:02.4" + rnP1 := "nvidia.com/sriov-port1" + rnP2 := "nvidia.com/sriov-port2" + + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + {Name: device1Name, Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &deviceID1}, multusResourceNameAttr: {StringValue: &rnP1}, + }}, + {Name: device2Name, Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &deviceID2}, multusResourceNameAttr: {StringValue: &rnP1}, + }}, + {Name: device3Name, Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &deviceID3}, multusResourceNameAttr: {StringValue: &rnP2}, + }}, + }, + }, + } + + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + {Request: "container-0-request-0", Driver: driverName, Pool: poolName, Device: device1Name}, + {Request: "container-0-request-0", Driver: driverName, Pool: poolName, Device: device2Name}, + {Request: "container-0-request-1", Driver: driverName, Pool: poolName, Device: device3Name}, + }, + }, + }, + }, + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multus-dual-port", + Namespace: "default", + UID: k8sTypes.UID("895bdfd9-a532-4127-97b5-2a0aa5cc91a7"), + }, + Status: v1.PodStatus{ + ExtendedResourceClaimStatus: &v1.PodExtendedResourceClaimStatus{ + ResourceClaimName: claimName, + RequestMappings: []v1.ContainerExtendedResourceRequest{ + {ContainerName: "test-container", ResourceName: "nvidia.com/sriov-port1", RequestName: "container-0-request-0"}, + {ContainerName: "test-container", ResourceName: "nvidia.com/sriov-port2", RequestName: "container-0-request-1"}, + }, + }, + }, + } + + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).NotTo(HaveOccurred()) + + Expect(resourceMap).To(HaveKey("nvidia.com/sriov-port1")) + Expect(resourceMap["nvidia.com/sriov-port1"].DeviceIDs).To(ConsistOf(deviceID1, deviceID2)) + Expect(resourceMap).To(HaveKey("nvidia.com/sriov-port2")) + Expect(resourceMap["nvidia.com/sriov-port2"].DeviceIDs).To(Equal([]string{deviceID3})) + }) + + It("should return an error when device resourceName attribute does not match extended mapping", func() { + claimName := "ext-mismatch-claim" + deviceName := "0000-08-00-3" + driverName := "sriovnetwork.k8snetworkplumbingwg.io" + poolName := "pool-1" + deviceID := "0000:08:00.3" + wrongRN := "wrong.example.com/port" + deviceIDVal := deviceID + + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{Name: "slice-mismatch"}, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1}, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &deviceIDVal}, + multusResourceNameAttr: {StringValue: &wrongRN}, + }, + }, + }, + }, + } + + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: claimName, Namespace: "default"}, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + {Request: "req-0", Driver: driverName, Pool: poolName, Device: deviceName}, + }, + }, + }, + }, + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-mismatch", Namespace: "default", UID: k8sTypes.UID("uid-mismatch")}, + Status: v1.PodStatus{ + ExtendedResourceClaimStatus: &v1.PodExtendedResourceClaimStatus{ + ResourceClaimName: claimName, + RequestMappings: []v1.ContainerExtendedResourceRequest{ + {ContainerName: "c", ResourceName: "expected.example.com/port", RequestName: "req-0"}, + }, + }, + }, + } + + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(multusResourceNameAttr)) + Expect(err.Error()).To(ContainSubstring("expected.example.com/port")) + }) + }) + + Context("when resource claim does not exist", func() { + It("should return an error", func() { + claimName := "non-existent-claim" + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: k8sTypes.UID("test-uid"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claimName, + ResourceClaimName: &claimNamePtr, + }, + }, + }, + } + + resourceMap := make(map[string]*types.ResourceInfo) + err := draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found")) + }) + }) + + Context("when resource slice does not exist", func() { + It("should return an error", func() { + claimName := "test-claim" + deviceName := "device-1" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + + // Create ResourceClaim but no ResourceSlice + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: k8sTypes.UID("test-uid"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claimName, + ResourceClaimName: &claimNamePtr, + }, + }, + }, + } + + // Add only resource claim + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Execute + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no resource slice found for driver/pool")) + }) + }) + + Context("when device does not have deviceID attribute", func() { + It("should return an error", func() { + claimName := "test-claim" + deviceName := "device-1" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + + // Create ResourceSlice with device but NO deviceID attribute + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + // Missing multusDeviceIDAttr + "some-other-attribute": { + StringValue: func() *string { s := "value"; return &s }(), + }, + }, + }, + }, + }, + } + + // Create ResourceClaim + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: k8sTypes.UID("test-uid"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claimName, + ResourceClaimName: &claimNamePtr, + }, + }, + }, + } + + // Add objects + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Execute + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found for claim resource")) + }) + }) + + Context("when device has deviceID but missing resourceName attribute", func() { + It("should return an error", func() { + claimName := "test-claim" + deviceName := "device-1" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + deviceID := "pci:0000:00:01.0" + deviceIDValue := deviceID + + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{Name: "test-resource-slice"}, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1}, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &deviceIDValue}, + }, + }, + }, + }, + } + + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: claimName, Namespace: "default"}, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + {Request: requestName, Driver: driverName, Pool: poolName, Device: deviceName}, + }, + }, + }, + }, + } + + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "default", UID: k8sTypes.UID("test-uid")}, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + {Name: claimName, ResourceClaimName: &claimNamePtr}, + }, + }, + } + + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(multusResourceNameAttr)) + }) + }) + + Context("when device name in allocation does not match any device in slice", func() { + It("should return an error", func() { + claimName := "test-claim" + deviceName := "device-1" + wrongDeviceName := "wrong-device" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + deviceID := "pci:0000:00:01.0" + + // Create ResourceSlice with device + deviceIDValue := deviceID + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: { + StringValue: &deviceIDValue, + }, + }, + }, + }, + }, + } + + // Create ResourceClaim with WRONG device name + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: wrongDeviceName, + }, + }, + }, + }, + }, + } + + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: k8sTypes.UID("test-uid"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claimName, + ResourceClaimName: &claimNamePtr, + }, + }, + }, + } + + // Add objects + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Execute + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found for claim resource")) + }) + }) + + Context("when caching is working correctly", func() { + It("should cache resource claims and slices", func() { + claimName := "test-claim" + deviceName := "device-1" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + deviceID := "pci:0000:00:01.0" + mapKey := "cache-test.example.com/res" + + // Create ResourceSlice with device + deviceIDValue := deviceID + mapKeyValue := mapKey + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: { + StringValue: &deviceIDValue, + }, + multusResourceNameAttr: {StringValue: &mapKeyValue}, + }, + }, + }, + }, + } + + // Create ResourceClaim + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: k8sTypes.UID("test-uid"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claimName, + ResourceClaimName: &claimNamePtr, + }, + }, + }, + } + + // Add objects to fake client + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // First call - should populate cache + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).NotTo(HaveOccurred()) + + expectedKey := mapKey + Expect(resourceMap).To(HaveKey(expectedKey)) + Expect(resourceMap[expectedKey].DeviceIDs).To(Equal([]string{deviceID})) + + // Delete the objects from the API server + err = fakeClient.ResourceV1().ResourceClaims("default").Delete(context.TODO(), claimName, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + err = fakeClient.ResourceV1().ResourceSlices().Delete(context.TODO(), resourceSlice.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Second call - should use cache and succeed even though objects are deleted + resourceMap2 := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap2) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap2).To(HaveKey(expectedKey)) + Expect(resourceMap2[expectedKey].DeviceIDs).To(Equal([]string{deviceID})) + }) + }) + + Context("when ResourceClaim names collide across namespaces", func() { + It("should cache claims separately per namespace", func() { + claimName := "same-claim-name" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + mapKey := "cross-ns.example.com/res" + nsA, nsB := "team-a", "team-b" + deviceA, deviceB := "dev-a", "dev-b" + pciA, pciB := "pci:0000:01:00.0", "pci:0000:02:00.0" + + mapKeyVal := mapKey + pciAVal, pciBVal := pciA, pciB + + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{Name: "shared-slice"}, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1}, + Devices: []resourcev1api.Device{ + { + Name: deviceA, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &pciAVal}, + multusResourceNameAttr: {StringValue: &mapKeyVal}, + }, + }, + { + Name: deviceB, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &pciBVal}, + multusResourceNameAttr: {StringValue: &mapKeyVal}, + }, + }, + }, + }, + } + + claimA := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: claimName, Namespace: nsA}, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + {Request: requestName, Driver: driverName, Pool: poolName, Device: deviceA}, + }, + }, + }, + }, + } + claimB := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: claimName, Namespace: nsB}, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + {Request: requestName, Driver: driverName, Pool: poolName, Device: deviceB}, + }, + }, + }, + }, + } + + _, err := fakeClient.ResourceV1().ResourceClaims(nsA).Create(context.TODO(), claimA, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceClaims(nsB).Create(context.TODO(), claimB, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + claimPtr := claimName + podA := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-a", Namespace: nsA, UID: k8sTypes.UID("uid-a")}, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + {Name: claimName, ResourceClaimName: &claimPtr}, + }, + }, + } + podB := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-b", Namespace: nsB, UID: k8sTypes.UID("uid-b")}, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + {Name: claimName, ResourceClaimName: &claimPtr}, + }, + }, + } + + m1 := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(podA, m1) + Expect(err).NotTo(HaveOccurred()) + Expect(m1[mapKey].DeviceIDs).To(Equal([]string{pciA})) + + m2 := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(podB, m2) + Expect(err).NotTo(HaveOccurred()) + Expect(m2[mapKey].DeviceIDs).To(Equal([]string{pciB})) + }) + }) + + Context("when pod has multiple different claims", func() { + It("should populate resource map with all claims sequentially", func() { + // Note: Due to fake client limitations with field selectors, + // we test each claim separately to avoid conflicts + + claim1Name := "claim-1" + device1Name := "device-1" + driver1Name := "driver1.example.com" + pool1Name := "pool-1" + request1Name := "gpu" + deviceID1 := "pci:0000:00:01.0" + mapKey1 := "driver1.example.com/gpu-net" + + // First claim setup + deviceID1Value := deviceID1 + mapKey1Val := mapKey1 + resourceSlice1 := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice-1", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driver1Name, + Pool: resourcev1api.ResourcePool{ + Name: pool1Name, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: device1Name, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: { + StringValue: &deviceID1Value, + }, + multusResourceNameAttr: {StringValue: &mapKey1Val}, + }, + }, + }, + }, + } + + resourceClaim1 := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claim1Name, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: request1Name, + Driver: driver1Name, + Pool: pool1Name, + Device: device1Name, + }, + }, + }, + }, + }, + } + + claim1NamePtr := claim1Name + pod1 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-1", + Namespace: "default", + UID: k8sTypes.UID("test-uid-1"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claim1Name, + ResourceClaimName: &claim1NamePtr, + }, + }, + }, + } + + // Add first claim objects + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim1, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice1, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Test first claim + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod1, resourceMap) + Expect(err).NotTo(HaveOccurred()) + + Expect(resourceMap).To(HaveKey(mapKey1)) + Expect(resourceMap[mapKey1].DeviceIDs).To(Equal([]string{deviceID1})) + + // Now test second claim with a fresh client to avoid field selector issues + claim2Name := "claim-2" + device2Name := "device-2" + driver2Name := "driver2.example.com" + pool2Name := "pool-2" + request2Name := "nic" + deviceID2 := "pci:0000:00:02.0" + mapKey2 := "driver2.example.com/nic-net" + + fakeClient2 := fake.NewSimpleClientset() + draClient2 := NewClient(fakeClient2.ResourceV1()) + + deviceID2Value := deviceID2 + mapKey2Val := mapKey2 + resourceSlice2 := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice-2", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driver2Name, + Pool: resourcev1api.ResourcePool{ + Name: pool2Name, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: device2Name, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: { + StringValue: &deviceID2Value, + }, + multusResourceNameAttr: {StringValue: &mapKey2Val}, + }, + }, + }, + }, + } + + resourceClaim2 := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claim2Name, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: request2Name, + Driver: driver2Name, + Pool: pool2Name, + Device: device2Name, + }, + }, + }, + }, + }, + } + + claim2NamePtr := claim2Name + pod2 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-2", + Namespace: "default", + UID: k8sTypes.UID("test-uid-2"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claim2Name, + ResourceClaimName: &claim2NamePtr, + }, + }, + }, + } + + // Add second claim objects + _, err = fakeClient2.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim2, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient2.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice2, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Test second claim + resourceMap2 := make(map[string]*types.ResourceInfo) + err = draClient2.GetPodResourceMap(pod2, resourceMap2) + Expect(err).NotTo(HaveOccurred()) + + Expect(resourceMap2).To(HaveKey(mapKey2)) + Expect(resourceMap2[mapKey2].DeviceIDs).To(Equal([]string{deviceID2})) + }) + }) + + Context("when resource map already has an entry for the claim/request", func() { + It("should append device IDs to existing entry", func() { + claimName := "test-claim" + deviceName := "device-1" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + deviceID := "pci:0000:00:01.0" + existingDeviceID := "pci:0000:00:00.0" + mapKey := "append-test.example.com/res" + + // Create ResourceSlice with device + deviceIDValue := deviceID + mapKeyValue := mapKey + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: { + StringValue: &deviceIDValue, + }, + multusResourceNameAttr: {StringValue: &mapKeyValue}, + }, + }, + }, + }, + } + + // Create ResourceClaim + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: k8sTypes.UID("test-uid"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claimName, + ResourceClaimName: &claimNamePtr, + }, + }, + }, + } + + // Add objects + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Pre-populate resourceMap with existing entry + resourceMap := make(map[string]*types.ResourceInfo) + expectedKey := mapKey + resourceMap[expectedKey] = &types.ResourceInfo{ + DeviceIDs: []string{existingDeviceID}, + } + + // Execute + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).NotTo(HaveOccurred()) + + // Verify device ID was appended + Expect(resourceMap).To(HaveKey(expectedKey)) + Expect(resourceMap[expectedKey].DeviceIDs).To(Equal([]string{existingDeviceID, deviceID})) + }) + }) + + Context("when multiple resource slices exist for the same driver/pool", func() { + It("should find the allocated device by searching all slices", func() { + claimName := "test-claim" + deviceName := "device-1" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + deviceID := "pci:0000:00:01.0" + mapKey := "multi-slice.example.com/res" + + // DRA allows multiple ResourceSlices per driver/pool (e.g. per NUMA). Allocation references device-1 + // which lives only in the first slice; second slice holds another device. + deviceIDValue := deviceID + mapKeyValue := mapKey + resourceSlice1 := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice-1", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: { + StringValue: &deviceIDValue, + }, + multusResourceNameAttr: {StringValue: &mapKeyValue}, + }, + }, + }, + }, + } + + resourceSlice2 := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice-2", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: "device-2", + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: { + StringValue: &deviceIDValue, + }, + multusResourceNameAttr: {StringValue: &mapKeyValue}, + }, + }, + }, + }, + } + + // Create ResourceClaim + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: "default", + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: k8sTypes.UID("test-uid"), + }, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + { + Name: claimName, + ResourceClaimName: &claimNamePtr, + }, + }, + }, + } + + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + var objects []runtime.Object + objects = append(objects, resourceSlice1, resourceSlice2) + fakeClient2 := fake.NewSimpleClientset(objects...) + draClient2 := NewClient(fakeClient2.ResourceV1()) + + _, err = fakeClient2.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient2.GetPodResourceMap(pod, resourceMap) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap).To(HaveKey(mapKey)) + Expect(resourceMap[mapKey].DeviceIDs).To(Equal([]string{deviceID})) + }) + }) + }) +}) diff --git a/pkg/k8sclient/k8sclient.go b/pkg/k8sclient/k8sclient.go index b809a5376..f2440f311 100644 --- a/pkg/k8sclient/k8sclient.go +++ b/pkg/k8sclient/k8sclient.go @@ -42,6 +42,7 @@ import ( netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" netlister "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils" + "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/draclient" "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/kubeletclient" "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging" "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types" @@ -53,6 +54,10 @@ const ( networkAttachmentAnnot = "k8s.v1.cni.cncf.io/networks" ) +// getResourceClientFunc returns kubelet / device-plugin resource info for a pod. +// It defaults to kubeletclient.GetResourceClient; unit tests may replace it to avoid a real kubelet checkpoint/socket. +var getResourceClientFunc = kubeletclient.GetResourceClient + // NoK8sNetworkError indicates error, no network in kubernetes type NoK8sNetworkError struct { message string @@ -309,7 +314,7 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName) if resourceMap == nil { - ck, err := kubeletclient.GetResourceClient("") + ck, err := getResourceClientFunc("") if err != nil { return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get a ResourceClient instance: %v", err) } @@ -317,6 +322,13 @@ func getKubernetesDelegate(client *ClientInfo, net *types.NetworkSelectionElemen if err != nil { return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from ResourceClient: %v", err) } + + dc := draclient.NewClient(client.Client.ResourceV1()) + err = dc.GetPodResourceMap(pod, resourceMap) + if err != nil { + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from DRA client: %v", err) + } + logging.Debugf("getKubernetesDelegate: resourceMap instance: %+v", resourceMap) } diff --git a/pkg/k8sclient/k8sclient_test.go b/pkg/k8sclient/k8sclient_test.go index 7e9a9332c..95a39fa99 100644 --- a/pkg/k8sclient/k8sclient_test.go +++ b/pkg/k8sclient/k8sclient_test.go @@ -18,6 +18,7 @@ package k8sclient // disable dot-imports only for testing //revive:disable:dot-imports import ( + "context" "fmt" "os" "path/filepath" @@ -33,6 +34,9 @@ import ( netfake "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" netutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils" + v1 "k8s.io/api/core/v1" + resourcev1api "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" . "github.com/onsi/ginkgo/v2" @@ -52,6 +56,13 @@ func NewFakeClientInfo() *ClientInfo { } } +// fakeEmptyResourceClient implements types.ResourceClient with no device allocations (stubs kubelet in tests). +type fakeEmptyResourceClient struct{} + +func (*fakeEmptyResourceClient) GetPodResourceMap(*v1.Pod) (map[string]*types.ResourceInfo, error) { + return make(map[string]*types.ResourceInfo), nil +} + var _ = Describe("k8sclient operations", func() { var tmpDir string var err error @@ -1553,4 +1564,572 @@ users: Expect(err).NotTo(HaveOccurred()) }) }) + + Describe("DRA (Dynamic Resource Allocation) integration", func() { + var tmpDir string + var err error + + BeforeEach(func() { + tmpDir, err = os.MkdirTemp("", "multus_dra_test") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := os.RemoveAll(tmpDir) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("when pod has DRA resources with device plugin resources", func() { + It("should combine DRA and device plugin resource maps", func() { + // This test verifies that getKubernetesDelegate correctly integrates + // DRA resources with traditional device plugin resources + + const fakePodName string = "dra-test-pod" + const fakeNamespace string = "default" + + // Create a network attachment definition + netAttachDef := `{ + "name": "sriov-net", + "type": "sriov", + "cniVersion": "0.3.1" + }` + + // Create fake pod with DRA resource claims + claimName := "gpu-claim" + claimNamePtr := &claimName + fakePod := testutils.NewFakePod(fakePodName, "sriov-net", "") + fakePod.Namespace = fakeNamespace + fakePod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{ + { + ResourceClaimName: claimNamePtr, + }, + } + + // Setup client + clientInfo := NewFakeClientInfo() + _, err = clientInfo.AddPod(fakePod) + Expect(err).NotTo(HaveOccurred()) + + // Create network attachment definition (without resourceName annotation to avoid kubeletclient) + nad := testutils.NewFakeNetAttachDef(fakeNamespace, "sriov-net", netAttachDef) + _, err = clientInfo.AddNetAttachDef(nad) + Expect(err).NotTo(HaveOccurred()) + + // Create ResourceClaim + deviceName := "gpu-1" + driverName := "gpu.example.com" + poolName := "gpu-pool" + requestName := "gpu" + deviceID := "pci:0000:00:01.0" + + deviceIDValue := deviceID + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + "k8s.cni.cncf.io/deviceID": { + StringValue: &deviceIDValue, + }, + }, + }, + }, + }, + } + + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: fakeNamespace, + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + // Add DRA resources to fake client + _, err = clientInfo.Client.ResourceV1().ResourceClaims(fakeNamespace).Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = clientInfo.Client.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Get the network selection element + net := &types.NetworkSelectionElement{ + Name: "sriov-net", + Namespace: fakeNamespace, + } + + // Call getKubernetesDelegate + // Note: Without resourceName annotation, DRA client is not invoked automatically + // This test verifies the delegate is created successfully for pods with DRA claims + delegate, resourceMap, err := getKubernetesDelegate(clientInfo, net, tmpDir, fakePod, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(delegate).NotTo(BeNil()) + + // Verify resourceMap is initialized (but empty since no resourceName annotation) + Expect(len(resourceMap)).To(Equal(0)) + }) + }) + + Context("when GetNetworkDelegates is called with DRA-enabled pod", func() { + It("should successfully retrieve delegates with DRA resources", func() { + const fakePodName string = "dra-network-pod" + const fakeNamespace string = "default" + + // Create network attachment definition + netAttachDef := `{ + "name": "dra-network", + "type": "bridge", + "cniVersion": "0.3.1" + }` + + // Create fake pod with DRA resource claims + claimName := "network-claim" + claimNamePtr := &claimName + fakePod := testutils.NewFakePod(fakePodName, "dra-network", "") + fakePod.Namespace = fakeNamespace + fakePod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{ + { + ResourceClaimName: claimNamePtr, + }, + } + + // Setup client + clientInfo := NewFakeClientInfo() + _, err = clientInfo.AddPod(fakePod) + Expect(err).NotTo(HaveOccurred()) + + // Create network attachment definition with resource name annotation + nad := testutils.NewFakeNetAttachDef(fakeNamespace, "dra-network", netAttachDef) + nad.Annotations = map[string]string{ + resourceNameAnnot: "dra-network-claim/gpu", + } + _, err = clientInfo.AddNetAttachDef(nad) + Expect(err).NotTo(HaveOccurred()) + + // Create DRA resources + deviceName := "nic-1" + driverName := "network.example.com" + poolName := "network-pool" + requestName := "gpu" + deviceID := "pci:0000:00:02.0" + // NAD k8s.v1.cni.cncf.io/resourceName must match device attribute k8s.cni.cncf.io/resourceName + nadResourceKey := "dra-network-claim/gpu" + + deviceIDValue := deviceID + nadResourceKeyVal := nadResourceKey + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "network-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + "k8s.cni.cncf.io/deviceID": { + StringValue: &deviceIDValue, + }, + "k8s.cni.cncf.io/resourceName": { + StringValue: &nadResourceKeyVal, + }, + }, + }, + }, + }, + } + + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: fakeNamespace, + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + // Add DRA resources to fake client + _, err = clientInfo.Client.ResourceV1().ResourceClaims(fakeNamespace).Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = clientInfo.Client.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Get pod network + networks, err := GetPodNetwork(fakePod) + Expect(err).NotTo(HaveOccurred()) + Expect(networks).NotTo(BeEmpty()) + + // Create NetConf + conf := &types.NetConf{ + ConfDir: tmpDir, + } + + // Get network delegates + resourceMap := make(map[string]*types.ResourceInfo) + delegates, err := GetNetworkDelegates(clientInfo, fakePod, networks, conf, resourceMap) + Expect(err).NotTo(HaveOccurred()) + Expect(delegates).NotTo(BeEmpty()) + Expect(delegates[0]).NotTo(BeNil()) + }) + }) + + Context("when DRA resources are used in TryLoadPodDelegates", func() { + It("should successfully load delegates with DRA resources", func() { + const fakePodName string = "dra-delegate-pod" + const fakeNamespace string = "default" + + // Create network attachment definition + netAttachDef := `{ + "name": "dra-delegate-network", + "type": "macvlan", + "cniVersion": "0.3.1" + }` + + // Create fake pod with DRA resource claims + claimName := "delegate-claim" + claimNamePtr := &claimName + fakePod := testutils.NewFakePod(fakePodName, "dra-delegate-network", "") + fakePod.Namespace = fakeNamespace + fakePod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{ + { + ResourceClaimName: claimNamePtr, + }, + } + + // Setup client + clientInfo := NewFakeClientInfo() + _, err = clientInfo.AddPod(fakePod) + Expect(err).NotTo(HaveOccurred()) + + // Create network attachment definition + nad := testutils.NewFakeNetAttachDef(fakeNamespace, "dra-delegate-network", netAttachDef) + _, err = clientInfo.AddNetAttachDef(nad) + Expect(err).NotTo(HaveOccurred()) + + // Create DRA resources + deviceName := "macvlan-device" + driverName := "macvlan.example.com" + poolName := "macvlan-pool" + requestName := "nic" + deviceID := "pci:0000:00:03.0" + + deviceIDValue := deviceID + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "macvlan-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + "k8s.cni.cncf.io/deviceID": { + StringValue: &deviceIDValue, + }, + }, + }, + }, + }, + } + + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: fakeNamespace, + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + // Add DRA resources to fake client + _, err = clientInfo.Client.ResourceV1().ResourceClaims(fakeNamespace).Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = clientInfo.Client.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Create NetConf with a default delegate + confStr := `{ + "name": "test-network", + "type": "multus", + "delegates": [{ + "name": "default-network", + "type": "bridge" + }], + "confDir": "` + tmpDir + `" + }` + + conf, err := types.LoadNetConf([]byte(confStr)) + Expect(err).NotTo(HaveOccurred()) + + // Try loading pod delegates + resourceMap := make(map[string]*types.ResourceInfo) + count, updatedClient, err := TryLoadPodDelegates(fakePod, conf, clientInfo, resourceMap) + Expect(err).NotTo(HaveOccurred()) + Expect(updatedClient).NotTo(BeNil()) + Expect(count).To(BeNumerically(">", 0)) + }) + }) + + Context("when pod has no DRA resources", func() { + It("should handle pods without DRA resources gracefully", func() { + const fakePodName string = "no-dra-pod" + const fakeNamespace string = "default" + + // Create network attachment definition + netAttachDef := `{ + "name": "simple-network", + "type": "bridge", + "cniVersion": "0.3.1" + }` + + // Create fake pod WITHOUT DRA resource claims + fakePod := testutils.NewFakePod(fakePodName, "simple-network", "") + fakePod.Namespace = fakeNamespace + fakePod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{} // Empty + + // Setup client + clientInfo := NewFakeClientInfo() + _, err = clientInfo.AddPod(fakePod) + Expect(err).NotTo(HaveOccurred()) + + // Create network attachment definition + nad := testutils.NewFakeNetAttachDef(fakeNamespace, "simple-network", netAttachDef) + _, err = clientInfo.AddNetAttachDef(nad) + Expect(err).NotTo(HaveOccurred()) + + // Get the network selection element + net := &types.NetworkSelectionElement{ + Name: "simple-network", + Namespace: fakeNamespace, + } + + // Call getKubernetesDelegate - should work without DRA + delegate, resourceMap, err := getKubernetesDelegate(clientInfo, net, tmpDir, fakePod, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(delegate).NotTo(BeNil()) + // ResourceMap should be empty (no DRA resources) + Expect(len(resourceMap)).To(Equal(0)) + }) + }) + + Context("when DRA client fails to get resources", func() { + var origGetResourceClient func(string) (types.ResourceClient, error) + + BeforeEach(func() { + origGetResourceClient = getResourceClientFunc + // Stub kubelet so we get past device-plugin checkpoint/socket and exercise draclient. + getResourceClientFunc = func(string) (types.ResourceClient, error) { + return &fakeEmptyResourceClient{}, nil + } + }) + + AfterEach(func() { + getResourceClientFunc = origGetResourceClient + }) + + It("should return an error from getKubernetesDelegate", func() { + const fakePodName string = "dra-fail-pod" + const fakeNamespace string = "default" + const resourceName = "example.com/gpu" + + // Create network attachment definition + netAttachDef := `{ + "name": "fail-network", + "type": "bridge", + "cniVersion": "0.3.1" + }` + + // Create fake pod with DRA resource claims that don't exist in the API + claimName := "non-existent-claim" + claimNamePtr := &claimName + fakePod := testutils.NewFakePod(fakePodName, "fail-network", "") + fakePod.Namespace = fakeNamespace + fakePod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{ + { + Name: claimName, + ResourceClaimName: claimNamePtr, + }, + } + + // Setup client + clientInfo := NewFakeClientInfo() + _, err = clientInfo.AddPod(fakePod) + Expect(err).NotTo(HaveOccurred()) + + // NAD must have resourceName annotation so getKubernetesDelegate runs the DRA client path + nad := testutils.NewFakeNetAttachDef(fakeNamespace, "fail-network", netAttachDef) + nad.Annotations = map[string]string{ + resourceNameAnnot: resourceName, + } + _, err = clientInfo.AddNetAttachDef(nad) + Expect(err).NotTo(HaveOccurred()) + + // Do NOT create the ResourceClaim — draclient.GetPodResourceMap fails on API Get. + + net := &types.NetworkSelectionElement{ + Name: "fail-network", + Namespace: fakeNamespace, + } + + _, _, err = getKubernetesDelegate(clientInfo, net, tmpDir, fakePod, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to get resourceMap from DRA client")) + Expect(err.Error()).To(ContainSubstring("not found")) + }) + }) + + Context("when using getNetDelegate with DRA resources", func() { + It("should retrieve delegates with DRA resources populated", func() { + const fakePodName string = "netdelegate-dra-pod" + const fakeNamespace string = "default" + + // Create network attachment definition + netAttachDef := `{ + "name": "netdelegate-network", + "type": "ipvlan", + "cniVersion": "0.3.1" + }` + + // Create fake pod with DRA resource claims + claimName := "netdelegate-claim" + claimNamePtr := &claimName + fakePod := testutils.NewFakePod(fakePodName, "", "") + fakePod.Namespace = fakeNamespace + fakePod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{ + { + ResourceClaimName: claimNamePtr, + }, + } + + // Setup client + clientInfo := NewFakeClientInfo() + _, err = clientInfo.AddPod(fakePod) + Expect(err).NotTo(HaveOccurred()) + + // Create network attachment definition + nad := testutils.NewFakeNetAttachDef(fakeNamespace, "netdelegate-network", netAttachDef) + _, err = clientInfo.AddNetAttachDef(nad) + Expect(err).NotTo(HaveOccurred()) + + // Create DRA resources + deviceName := "ipvlan-device" + driverName := "ipvlan.example.com" + poolName := "ipvlan-pool" + requestName := "vnic" + deviceID := "pci:0000:00:04.0" + + deviceIDValue := deviceID + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipvlan-resource-slice", + }, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{ + Name: poolName, + ResourceSliceCount: 1, + }, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + "k8s.cni.cncf.io/deviceID": { + StringValue: &deviceIDValue, + }, + }, + }, + }, + }, + } + + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: fakeNamespace, + }, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + { + Request: requestName, + Driver: driverName, + Pool: poolName, + Device: deviceName, + }, + }, + }, + }, + }, + } + + // Add DRA resources to fake client + _, err = clientInfo.Client.ResourceV1().ResourceClaims(fakeNamespace).Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = clientInfo.Client.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Call getNetDelegate + resourceMap := make(map[string]*types.ResourceInfo) + delegate, updatedResourceMap, err := getNetDelegate(clientInfo, fakePod, "netdelegate-network", tmpDir, fakeNamespace, resourceMap) + Expect(err).NotTo(HaveOccurred()) + Expect(delegate).NotTo(BeNil()) + Expect(updatedResourceMap).NotTo(BeNil()) + }) + }) + }) }) diff --git a/pkg/kubeletclient/kubeletclient.go b/pkg/kubeletclient/kubeletclient.go index 0b7bc5d2a..5398f1802 100644 --- a/pkg/kubeletclient/kubeletclient.go +++ b/pkg/kubeletclient/kubeletclient.go @@ -21,7 +21,6 @@ import ( "net/url" "os" "path/filepath" - "strings" "time" "golang.org/x/net/context" @@ -111,7 +110,6 @@ type kubeletClient struct { } func (rc *kubeletClient) getPodResources(client podresourcesapi.PodResourcesListerClient) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -139,7 +137,6 @@ func (rc *kubeletClient) GetPodResourceMap(pod *v1.Pod) (map[string]*types.Resou if pr.Name == name && pr.Namespace == ns { for _, cnt := range pr.Containers { rc.getDevicePluginResources(cnt.Devices, resourceMap) - rc.getDRAResources(cnt.DynamicResources, resourceMap) } } } @@ -157,27 +154,6 @@ func (rc *kubeletClient) getDevicePluginResources(devices []*podresourcesapi.Con } } -func (rc *kubeletClient) getDRAResources(dynamicResources []*podresourcesapi.DynamicResource, resourceMap map[string]*types.ResourceInfo) { - for _, dynamicResource := range dynamicResources { - var deviceIDs []string - for _, claimResource := range dynamicResource.ClaimResources { - for _, cdiDevice := range claimResource.CdiDevices { - res := strings.Split(cdiDevice.Name, "=") - if len(res) == 2 { - deviceIDs = append(deviceIDs, res[1]) - } else { - logging.Errorf("GetPodResourceMap: Invalid CDI format") - } - } - } - if rInfo, ok := resourceMap[dynamicResource.ClaimName]; ok { - rInfo.DeviceIDs = append(rInfo.DeviceIDs, deviceIDs...) - } else { - resourceMap[dynamicResource.ClaimName] = &types.ResourceInfo{DeviceIDs: deviceIDs} - } - } -} - func hasKubeletAPIEndpoint(url *url.URL) bool { // Check for kubelet resource API socket file if _, err := os.Stat(url.Path); err != nil { diff --git a/pkg/kubeletclient/kubeletclient_test.go b/pkg/kubeletclient/kubeletclient_test.go index a9593a1c1..7661fcb7e 100644 --- a/pkg/kubeletclient/kubeletclient_test.go +++ b/pkg/kubeletclient/kubeletclient_test.go @@ -46,8 +46,8 @@ var ( ) type fakeResourceServer struct { - server *grpc.Server podresourcesapi.UnimplementedPodResourcesListerServer + server *grpc.Server } // TODO: This is stub code for test, but we may need to change for the testing we use this API in the future... @@ -81,7 +81,7 @@ func (m *fakeResourceServer) List(_ context.Context, _ *podresourcesapi.ListPodR { CdiDevices: cdiDevices, DriverName: draDriverName, - PoolName: poolName, + PoolName: poolName, DeviceName: deviceName, }, } @@ -249,34 +249,6 @@ var _ = Describe("Kubelet resource endpoint data read operations", func() { Expect(resourceMap).To(Equal(outputRMap)) }) - It("should return no error with dynamic resource", func() { - podUID := k8sTypes.UID("9f94e27b-4233-43d6-bd10-f73b4de6f456") - fakePod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dynamic-resource-pod-name", - Namespace: "dynamic-resource-pod-namespace", - UID: podUID, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dynamic-resource-container-name", - }, - }, - }, - } - client, err := getKubeletClient(testKubeletSocket) - Expect(err).NotTo(HaveOccurred()) - - outputRMap := map[string]*mtypes.ResourceInfo{ - "resource-claim": {DeviceIDs: []string{"cdi-resource"}}, - } - resourceMap, err := client.GetPodResourceMap(fakePod) - Expect(err).NotTo(HaveOccurred()) - Expect(resourceMap).ShouldNot(BeNil()) - Expect(resourceMap).To(Equal(outputRMap)) - }) - It("should return an error with garbage socket value", func() { u, err := url.Parse("/badfilepath!?//") Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/logging/logging.go b/pkg/logging/logging.go index 5996a07af..b5c18209d 100644 --- a/pkg/logging/logging.go +++ b/pkg/logging/logging.go @@ -138,6 +138,11 @@ func Errorf(format string, a ...interface{}) error { return fmt.Errorf(format, a...) } +// Warningf prints a warning at the same visibility threshold as Errorf (prefix "warning:"). +func Warningf(format string, a ...interface{}) { + printf(ErrorLevel, "warning: "+format, a...) +} + // Panicf prints logging plus stack trace. This should be used only for unrecoverable error func Panicf(format string, a ...interface{}) { printf(PanicLevel, format, a...) diff --git a/pkg/multus/multus.go b/pkg/multus/multus.go index a54854ba4..1dedf8575 100644 --- a/pkg/multus/multus.go +++ b/pkg/multus/multus.go @@ -622,7 +622,7 @@ func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *type // Check if we had any errors, and send them all back. if len(errorstrings) > 0 { - return fmt.Errorf("%s", strings.Join(errorstrings, " / ")) + return fmt.Errorf("errors: %s", strings.Join(errorstrings, " / ")) } return nil From 42212e292ee9fc6c84971ce3e1b4947af39846c9 Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Mon, 6 Apr 2026 07:38:02 +0300 Subject: [PATCH 2/4] draclient: do not fail CNI when a DRA claim contributes nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - If every allocation result for a claim is skipped (no slice match, missing Multus deviceID/resourceName, etc.), log a warning and continue to the next claim instead of returning an error, so kubelet/device-plugin entries stay usable (hybrid legacy VF + broken or irrelevant DRA claims). - getDeviceInfo: treat missing multus deviceID on a matched device as the same skippable sentinel as “not in slice”, avoiding a misleading final error. - Tests: expect nil error for unmapped claims; add case preserving pre-filled resource map entries; keep VF+GPU success with empty GPU slice. - k8sclient: stub kubelet ResourceClient in DRA failure test; logging: Warningf. Signed-off-by: Fred Rolland --- pkg/draclient/draclient.go | 44 +++++++-- pkg/draclient/draclient_test.go | 155 ++++++++++++++++++++++++++++++-- 2 files changed, 181 insertions(+), 18 deletions(-) diff --git a/pkg/draclient/draclient.go b/pkg/draclient/draclient.go index a11b72fef..a8791d5d7 100644 --- a/pkg/draclient/draclient.go +++ b/pkg/draclient/draclient.go @@ -16,6 +16,7 @@ package draclient import ( "context" + "errors" "fmt" "time" @@ -33,6 +34,11 @@ const ( multusResourceNameAttr = "k8s.cni.cncf.io/resourceName" ) +// errDeviceNotInAnySlice is returned when allocation names a device that does not appear +// in any ResourceSlice for that driver/pool (wrapped in getDeviceInfo). Callers may skip +// individual results so multi-device claims (e.g. SR-IOV + GPU) still succeed for CNI. +var errDeviceNotInAnySlice = errors.New("device not in any matching resource slice") + // namespacedClaimCacheKey avoids cache collisions: ResourceClaim is namespaced. func namespacedClaimCacheKey(namespace, claimName string) string { return namespace + "/" + claimName @@ -99,20 +105,29 @@ func (d *draClient) GetPodResourceMap(pod *v1.Pod, resourceMap map[string]*types return fmt.Errorf("claim %s has no device allocation", claimName) } - for _, result := range resourceClaim.Status.Allocation.Devices.Results { + results := resourceClaim.Status.Allocation.Devices.Results + resolvedCount := 0 + for _, result := range results { logging.Debugf("GetPodResourceMap: processing device allocation - driver: %s, pool: %s, device: %s, request: %s", result.Driver, result.Pool, result.Device, result.Request) info, err := d.getDeviceInfo(ctx, result) if err != nil { + if errors.Is(err, errDeviceNotInAnySlice) { + logging.Warningf( + "GetPodResourceMap: skipping allocation result for claim %s (driver=%s pool=%s device=%s): %v", + claimName, result.Driver, result.Pool, result.Device, err) + continue + } logging.Errorf("GetPodResourceMap: failed to get device info for claim %s: %v", claimName, err) return err } if info.ResourceName == "" { - resErr := fmt.Errorf("device %s missing required attribute %s (must match NAD k8s.v1.cni.cncf.io/resourceName)", result.Device, multusResourceNameAttr) - logging.Errorf("GetPodResourceMap: %v", resErr) - return resErr + logging.Warningf( + "GetPodResourceMap: skipping allocation result for claim %s (driver=%s pool=%s device=%s): no %q (only devices published for CNI are mapped)", + claimName, result.Driver, result.Pool, result.Device, multusResourceNameAttr) + continue } resourceMapKey := info.ResourceName @@ -123,6 +138,14 @@ func (d *draClient) GetPodResourceMap(pod *v1.Pod, resourceMap map[string]*types resourceMap[resourceMapKey] = &types.ResourceInfo{DeviceIDs: []string{info.DeviceID}} logging.Debugf("GetPodResourceMap: created new resource map entry %s with device ID %s", resourceMapKey, info.DeviceID) } + resolvedCount++ + } + if resolvedCount == 0 && len(results) > 0 { + logging.Warningf( + "GetPodResourceMap: claim %s had no allocation results mapped for Multus (skipping this claim; existing kubelet/device-plugin map entries are kept). "+ + "Fix DRA ResourceSlices or Multus attributes if this claim should contribute to CNI.", + claimName) + continue } logging.Debugf("GetPodResourceMap: successfully processed resource claim %s", claimName) } @@ -259,16 +282,18 @@ func (d *draClient) getDeviceInfo(ctx context.Context, result resourcev1api.Devi devIDAttr, exists := device.Attributes[multusDeviceIDAttr] if !exists { logging.Warningf( - "getDeviceInfo: allocated device %q (driver %q, pool %q) has no %q attribute; DRA drivers must publish this for Multus; skipping device", + "getDeviceInfo: device %q (driver %q, pool %q) has no %q in ResourceSlice; skipping allocation result", device.Name, result.Driver, result.Pool, multusDeviceIDAttr) - continue + return nil, fmt.Errorf("%w: device %q present in slice but missing %q", + errDeviceNotInAnySlice, device.Name, multusDeviceIDAttr) } if devIDAttr.StringValue == nil { logging.Warningf( - "getDeviceInfo: allocated device %q (driver %q, pool %q) has %q with nil StringValue; skipping device", + "getDeviceInfo: device %q (driver %q, pool %q) has %q with nil StringValue; skipping allocation result", device.Name, result.Driver, result.Pool, multusDeviceIDAttr) - continue + return nil, fmt.Errorf("%w: device %q has nil StringValue for %q", + errDeviceNotInAnySlice, device.Name, multusDeviceIDAttr) } info := &deviceInfo{DeviceID: *devIDAttr.StringValue} @@ -283,7 +308,8 @@ func (d *draClient) getDeviceInfo(ctx context.Context, result resourcev1api.Devi } } - notFoundErr := fmt.Errorf("device %s not found for claim resource %s/%s in any matching resource slice", result.Device, result.Driver, result.Pool) + notFoundErr := fmt.Errorf("%w: device %s not found for claim resource %s/%s in any matching resource slice", + errDeviceNotInAnySlice, result.Device, result.Driver, result.Pool) logging.Errorf("getDeviceInfo: %v", notFoundErr) return nil, notFoundErr } diff --git a/pkg/draclient/draclient_test.go b/pkg/draclient/draclient_test.go index e88178de7..ba0788bd7 100644 --- a/pkg/draclient/draclient_test.go +++ b/pkg/draclient/draclient_test.go @@ -705,7 +705,7 @@ var _ = Describe("DRA Client operations", func() { }) Context("when device does not have deviceID attribute", func() { - It("should return an error", func() { + It("should warn and skip the claim without failing", func() { claimName := "test-claim" deviceName := "device-1" driverName := "test-driver.example.com" @@ -785,13 +785,13 @@ var _ = Describe("DRA Client operations", func() { // Execute resourceMap := make(map[string]*types.ResourceInfo) err = draClient.GetPodResourceMap(pod, resourceMap) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("not found for claim resource")) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap).To(BeEmpty()) }) }) Context("when device has deviceID but missing resourceName attribute", func() { - It("should return an error", func() { + It("should warn and skip without failing when nothing maps to a resource name", func() { claimName := "test-claim" deviceName := "device-1" driverName := "test-driver.example.com" @@ -846,13 +846,13 @@ var _ = Describe("DRA Client operations", func() { resourceMap := make(map[string]*types.ResourceInfo) err = draClient.GetPodResourceMap(pod, resourceMap) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(multusResourceNameAttr)) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap).To(BeEmpty()) }) }) Context("when device name in allocation does not match any device in slice", func() { - It("should return an error", func() { + It("should warn and skip the claim without failing", func() { claimName := "test-claim" deviceName := "device-1" wrongDeviceName := "wrong-device" @@ -934,8 +934,145 @@ var _ = Describe("DRA Client operations", func() { // Execute resourceMap := make(map[string]*types.ResourceInfo) err = draClient.GetPodResourceMap(pod, resourceMap) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("not found for claim resource")) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap).To(BeEmpty()) + }) + + It("should preserve existing kubelet map entries when the claim maps nothing", func() { + claimName := "test-claim" + deviceName := "device-1" + wrongDeviceName := "wrong-device" + driverName := "test-driver.example.com" + poolName := "test-pool" + requestName := "gpu" + deviceID := "pci:0000:00:01.0" + legacyKey := "example.com/legacy-vf" + legacyPCI := "0000:8d:00.4" + + deviceIDValue := deviceID + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{Name: "test-resource-slice"}, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverName, + Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1}, + Devices: []resourcev1api.Device{ + { + Name: deviceName, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &deviceIDValue}, + }, + }, + }, + }, + } + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: claimName, Namespace: "default"}, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + {Request: requestName, Driver: driverName, Pool: poolName, Device: wrongDeviceName}, + }, + }, + }, + }, + } + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "hybrid-pod", Namespace: "default", UID: k8sTypes.UID("uid-hybrid")}, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + {Name: claimName, ResourceClaimName: &claimNamePtr}, + }, + }, + } + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + resourceMap := map[string]*types.ResourceInfo{ + legacyKey: {DeviceIDs: []string{legacyPCI}}, + } + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap[legacyKey].DeviceIDs).To(Equal([]string{legacyPCI})) + }) + + It("should succeed when one allocation result is missing from slices but another resolves", func() { + claimName := "multi-claim" + driverSRIOV := "sriovnetwork.k8snetworkplumbingwg.io" + driverGPU := "gpu.nvidia.com" + poolName := "orch-dev-a100-002" + deviceVF := "0000-8d-00-4" + deviceGPU := "gpu-4" + deviceIDVF := "pci:0000:8d:00.4" + mapKey := "nvidia.com/port2" + deviceIDVFVal := deviceIDVF + mapKeyVal := mapKey + + // SR-IOV slice has the VF; empty GPU slice exists so List() finds gpu.nvidia.com/pool (real clusters + // always publish a slice per driver/pool). Allocation still references gpu-4, which is not listed here. + resourceSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{Name: "sriov-slice"}, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverSRIOV, + Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1}, + Devices: []resourcev1api.Device{ + { + Name: deviceVF, + Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{ + multusDeviceIDAttr: {StringValue: &deviceIDVFVal}, + multusResourceNameAttr: {StringValue: &mapKeyVal}, + }, + }, + }, + }, + } + gpuSlice := &resourcev1api.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{Name: "gpu-slice"}, + Spec: resourcev1api.ResourceSliceSpec{ + Driver: driverGPU, + Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1}, + Devices: []resourcev1api.Device{}, + }, + } + + resourceClaim := &resourcev1api.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: claimName, Namespace: "default"}, + Status: resourcev1api.ResourceClaimStatus{ + Allocation: &resourcev1api.AllocationResult{ + Devices: resourcev1api.DeviceAllocationResult{ + Results: []resourcev1api.DeviceRequestAllocationResult{ + {Request: "vf", Driver: driverSRIOV, Pool: poolName, Device: deviceVF}, + {Request: "gpu", Driver: driverGPU, Pool: poolName, Device: deviceGPU}, + }, + }, + }, + }, + } + + claimNamePtr := claimName + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "mixed-pod", Namespace: "default", UID: k8sTypes.UID("uid-mixed")}, + Status: v1.PodStatus{ + ResourceClaimStatuses: []v1.PodResourceClaimStatus{ + {Name: claimName, ResourceClaimName: &claimNamePtr}, + }, + }, + } + + _, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), gpuSlice, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + resourceMap := make(map[string]*types.ResourceInfo) + err = draClient.GetPodResourceMap(pod, resourceMap) + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap[mapKey].DeviceIDs).To(Equal([]string{deviceIDVF})) }) }) From 0913b1c3a18687b37fe8a946d539bb0580bc3072 Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Sun, 26 Apr 2026 17:33:26 +0300 Subject: [PATCH 3/4] draclient: use field selectors and lightweight device index for ResourceSlice cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace full ResourceSlice object cache with a flat deviceInfoCacheKey→deviceInfo index. Populate lazily per driver using combined spec.nodeName+spec.driver field selectors derived from pod.Spec.NodeName, reducing API response size to a single node's slices for one driver. Device lookup becomes O(1). Add ctx propagation to GetPodResourceMap interface. Signed-off-by: Fred Rolland --- docs/how-to-use.md | 23 +- pkg/draclient/draclient.go | 258 ++++++++------- pkg/draclient/draclient_test.go | 402 ++++++++++++++++++++---- pkg/k8sclient/k8sclient.go | 4 +- pkg/k8sclient/k8sclient_test.go | 288 ++++++++++++----- pkg/kubeletclient/kubeletclient_test.go | 36 --- 6 files changed, 729 insertions(+), 282 deletions(-) diff --git a/docs/how-to-use.md b/docs/how-to-use.md index 58f1efa04..7bd696926 100644 --- a/docs/how-to-use.md +++ b/docs/how-to-use.md @@ -658,19 +658,24 @@ concepts to make use of Multus DRA support. #### Install DRA driver -You need to install a DRA driver that provides network devices. For example, you can use the SR-IOV DRA driver or +You need to install a DRA driver that provides network devices. For example, you can use the [SR-IOV DRA](https://github.com/k8snetworkplumbingwg/dra-driver-sriov) driver or other DRA networking drivers. Refer to your DRA driver documentation for installation instructions. -The DRA driver MUST expose the following attributes on each allocated **device** in `ResourceSlice`: +The DRA driver MUST expose the following attributes on each allocated **network** device in `ResourceSlice`: - `k8s.cni.cncf.io/deviceID`: device ID that Multus passes to the CNI plugin. - `k8s.cni.cncf.io/resourceName`: **must exactly match** the value you put on the NetworkAttachmentDefinition `k8s.v1.cni.cncf.io/resourceName` annotation (same style as classic extended resources, e.g. `intel.com/sriov_vf`). - If `k8s.cni.cncf.io/resourceName` is missing on the device, allocation processing fails. + +Devices that lack `k8s.cni.cncf.io/deviceID` or `k8s.cni.cncf.io/resourceName` are **silently skipped** in the +normal claims path — this is intentional so that mixed pods with both network and non-network DRA claims (e.g. a +GPU claim alongside an SR-IOV claim) work correctly. The existing kubelet / device-plugin resource-map entries for +such pods are preserved unchanged. For pods that use the **extended resource** feature gate, Multus uses `pod.status.extendedResourceClaimStatus` -request mappings: the NAD `resourceName` matches `requestMappings[].resourceName`. The device attribute -`k8s.cni.cncf.io/resourceName` must be set and must **equal** that same `requestMappings[].resourceName` value -(Multus rejects a mismatch). Device lookup uses the same `ResourceClaim` / `ResourceSlice` flow. +request mappings: the NAD `resourceName` matches `requestMappings[].resourceName`. In this path the device +attribute `k8s.cni.cncf.io/resourceName` **must** be present and **equal** `requestMappings[].resourceName` +exactly — a missing or mismatched value is treated as an error, because the extended-resource mapping explicitly +names the expected device. Device lookup uses the same `ResourceClaim` / `ResourceSlice` flow as the normal path. #### Create network attachment definition with resource name @@ -732,11 +737,11 @@ cat < Date: Wed, 24 Jun 2026 16:31:20 +0300 Subject: [PATCH 4/4] server: fix ignoreReadinessIndicator being silently skipped when no server config is set overrideCNIConfigWithServerConfig returned the client config unchanged whenever overrideConf was empty, bypassing the readinessindicatorfile key-stripping logic even when ignoreReadinessIndicator=true. This caused STATUS and GC handlers to poll for the readiness indicator file for up to 45 s in tests that create the server with ignoreReadinessIndicator=true and no server-side config override. Also remove the now-unused sets import. Signed-off-by: Fred Rolland --- pkg/server/server.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index a7d81ebbd..6163539b8 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -51,7 +51,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/sets" utilwait "k8s.io/apimachinery/pkg/util/wait" informerfactory "k8s.io/client-go/informers" v1coreinformers "k8s.io/client-go/informers/core/v1" @@ -479,9 +478,16 @@ func (s *Server) handleDelegateRequest(r *http.Request) ([]byte, error) { } func overrideCNIConfigWithServerConfig(cniConf []byte, overrideConf []byte, ignoreReadinessIndicator bool) ([]byte, error) { - if len(overrideConf) == 0 { + // If there is no server-side override config AND we don't need to strip any keys, + // return the client config unchanged. + if len(overrideConf) == 0 && !ignoreReadinessIndicator { return cniConf, nil } + // Treat a missing server config as an empty object so the key-stripping logic below + // still runs when ignoreReadinessIndicator is true. + if len(overrideConf) == 0 { + overrideConf = []byte("{}") + } var cni map[string]interface{} if err := json.Unmarshal(cniConf, &cni); err != nil { @@ -493,14 +499,13 @@ func overrideCNIConfigWithServerConfig(cniConf []byte, overrideConf []byte, igno return nil, fmt.Errorf("failed to unmarshall CNI override config: %w", err) } - // Copy each key of the override config into the CNI config except for - // a few specific keys - ignoreKeys := sets.NewString() + // Remove keys from the client config that the server wants to ignore, then + // overlay the server-side overrides (also skipping those same keys). if ignoreReadinessIndicator { - ignoreKeys.Insert("readinessindicatorfile") + delete(cni, "readinessindicatorfile") } for overrideKey, overrideVal := range override { - if !ignoreKeys.Has(overrideKey) { + if !ignoreReadinessIndicator || overrideKey != "readinessindicatorfile" { cni[overrideKey] = overrideVal } }