mirror of
https://github.com/k8snetworkplumbingwg/multus-cni.git
synced 2026-07-16 17:17:15 +00:00
draclient: do not fail CNI when a DRA claim contributes nothing
- If every allocation result for a claim is skipped (no slice match, missing Multus deviceID/resourceName, etc.), log a warning and continue to the next claim instead of returning an error, so kubelet/device-plugin entries stay usable (hybrid legacy VF + broken or irrelevant DRA claims). - getDeviceInfo: treat missing multus deviceID on a matched device as the same skippable sentinel as “not in slice”, avoiding a misleading final error. - Tests: expect nil error for unmapped claims; add case preserving pre-filled resource map entries; keep VF+GPU success with empty GPU slice. - k8sclient: stub kubelet ResourceClient in DRA failure test; logging: Warningf. Signed-off-by: Fred Rolland <frolland@nvidia.com>
This commit is contained in:
Fred Rolland
@@ -16,6 +16,7 @@ package draclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -33,6 +34,11 @@ const (
|
||||
multusResourceNameAttr = "k8s.cni.cncf.io/resourceName"
|
||||
)
|
||||
|
||||
// errDeviceNotInAnySlice is returned when allocation names a device that does not appear
|
||||
// in any ResourceSlice for that driver/pool (wrapped in getDeviceInfo). Callers may skip
|
||||
// individual results so multi-device claims (e.g. SR-IOV + GPU) still succeed for CNI.
|
||||
var errDeviceNotInAnySlice = errors.New("device not in any matching resource slice")
|
||||
|
||||
// namespacedClaimCacheKey avoids cache collisions: ResourceClaim is namespaced.
|
||||
func namespacedClaimCacheKey(namespace, claimName string) string {
|
||||
return namespace + "/" + claimName
|
||||
@@ -99,20 +105,29 @@ func (d *draClient) GetPodResourceMap(pod *v1.Pod, resourceMap map[string]*types
|
||||
return fmt.Errorf("claim %s has no device allocation", claimName)
|
||||
}
|
||||
|
||||
for _, result := range resourceClaim.Status.Allocation.Devices.Results {
|
||||
results := resourceClaim.Status.Allocation.Devices.Results
|
||||
resolvedCount := 0
|
||||
for _, result := range results {
|
||||
logging.Debugf("GetPodResourceMap: processing device allocation - driver: %s, pool: %s, device: %s, request: %s",
|
||||
result.Driver, result.Pool, result.Device, result.Request)
|
||||
|
||||
info, err := d.getDeviceInfo(ctx, result)
|
||||
if err != nil {
|
||||
if errors.Is(err, errDeviceNotInAnySlice) {
|
||||
logging.Warningf(
|
||||
"GetPodResourceMap: skipping allocation result for claim %s (driver=%s pool=%s device=%s): %v",
|
||||
claimName, result.Driver, result.Pool, result.Device, err)
|
||||
continue
|
||||
}
|
||||
logging.Errorf("GetPodResourceMap: failed to get device info for claim %s: %v", claimName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if info.ResourceName == "" {
|
||||
resErr := fmt.Errorf("device %s missing required attribute %s (must match NAD k8s.v1.cni.cncf.io/resourceName)", result.Device, multusResourceNameAttr)
|
||||
logging.Errorf("GetPodResourceMap: %v", resErr)
|
||||
return resErr
|
||||
logging.Warningf(
|
||||
"GetPodResourceMap: skipping allocation result for claim %s (driver=%s pool=%s device=%s): no %q (only devices published for CNI are mapped)",
|
||||
claimName, result.Driver, result.Pool, result.Device, multusResourceNameAttr)
|
||||
continue
|
||||
}
|
||||
|
||||
resourceMapKey := info.ResourceName
|
||||
@@ -123,6 +138,14 @@ func (d *draClient) GetPodResourceMap(pod *v1.Pod, resourceMap map[string]*types
|
||||
resourceMap[resourceMapKey] = &types.ResourceInfo{DeviceIDs: []string{info.DeviceID}}
|
||||
logging.Debugf("GetPodResourceMap: created new resource map entry %s with device ID %s", resourceMapKey, info.DeviceID)
|
||||
}
|
||||
resolvedCount++
|
||||
}
|
||||
if resolvedCount == 0 && len(results) > 0 {
|
||||
logging.Warningf(
|
||||
"GetPodResourceMap: claim %s had no allocation results mapped for Multus (skipping this claim; existing kubelet/device-plugin map entries are kept). "+
|
||||
"Fix DRA ResourceSlices or Multus attributes if this claim should contribute to CNI.",
|
||||
claimName)
|
||||
continue
|
||||
}
|
||||
logging.Debugf("GetPodResourceMap: successfully processed resource claim %s", claimName)
|
||||
}
|
||||
@@ -259,16 +282,18 @@ func (d *draClient) getDeviceInfo(ctx context.Context, result resourcev1api.Devi
|
||||
devIDAttr, exists := device.Attributes[multusDeviceIDAttr]
|
||||
if !exists {
|
||||
logging.Warningf(
|
||||
"getDeviceInfo: allocated device %q (driver %q, pool %q) has no %q attribute; DRA drivers must publish this for Multus; skipping device",
|
||||
"getDeviceInfo: device %q (driver %q, pool %q) has no %q in ResourceSlice; skipping allocation result",
|
||||
device.Name, result.Driver, result.Pool, multusDeviceIDAttr)
|
||||
continue
|
||||
return nil, fmt.Errorf("%w: device %q present in slice but missing %q",
|
||||
errDeviceNotInAnySlice, device.Name, multusDeviceIDAttr)
|
||||
}
|
||||
|
||||
if devIDAttr.StringValue == nil {
|
||||
logging.Warningf(
|
||||
"getDeviceInfo: allocated device %q (driver %q, pool %q) has %q with nil StringValue; skipping device",
|
||||
"getDeviceInfo: device %q (driver %q, pool %q) has %q with nil StringValue; skipping allocation result",
|
||||
device.Name, result.Driver, result.Pool, multusDeviceIDAttr)
|
||||
continue
|
||||
return nil, fmt.Errorf("%w: device %q has nil StringValue for %q",
|
||||
errDeviceNotInAnySlice, device.Name, multusDeviceIDAttr)
|
||||
}
|
||||
info := &deviceInfo{DeviceID: *devIDAttr.StringValue}
|
||||
|
||||
@@ -283,7 +308,8 @@ func (d *draClient) getDeviceInfo(ctx context.Context, result resourcev1api.Devi
|
||||
}
|
||||
}
|
||||
|
||||
notFoundErr := fmt.Errorf("device %s not found for claim resource %s/%s in any matching resource slice", result.Device, result.Driver, result.Pool)
|
||||
notFoundErr := fmt.Errorf("%w: device %s not found for claim resource %s/%s in any matching resource slice",
|
||||
errDeviceNotInAnySlice, result.Device, result.Driver, result.Pool)
|
||||
logging.Errorf("getDeviceInfo: %v", notFoundErr)
|
||||
return nil, notFoundErr
|
||||
}
|
||||
|
||||
@@ -705,7 +705,7 @@ var _ = Describe("DRA Client operations", func() {
|
||||
})
|
||||
|
||||
Context("when device does not have deviceID attribute", func() {
|
||||
It("should return an error", func() {
|
||||
It("should warn and skip the claim without failing", func() {
|
||||
claimName := "test-claim"
|
||||
deviceName := "device-1"
|
||||
driverName := "test-driver.example.com"
|
||||
@@ -785,13 +785,13 @@ var _ = Describe("DRA Client operations", func() {
|
||||
// Execute
|
||||
resourceMap := make(map[string]*types.ResourceInfo)
|
||||
err = draClient.GetPodResourceMap(pod, resourceMap)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("not found for claim resource"))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(resourceMap).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("when device has deviceID but missing resourceName attribute", func() {
|
||||
It("should return an error", func() {
|
||||
It("should warn and skip without failing when nothing maps to a resource name", func() {
|
||||
claimName := "test-claim"
|
||||
deviceName := "device-1"
|
||||
driverName := "test-driver.example.com"
|
||||
@@ -846,13 +846,13 @@ var _ = Describe("DRA Client operations", func() {
|
||||
|
||||
resourceMap := make(map[string]*types.ResourceInfo)
|
||||
err = draClient.GetPodResourceMap(pod, resourceMap)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring(multusResourceNameAttr))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(resourceMap).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("when device name in allocation does not match any device in slice", func() {
|
||||
It("should return an error", func() {
|
||||
It("should warn and skip the claim without failing", func() {
|
||||
claimName := "test-claim"
|
||||
deviceName := "device-1"
|
||||
wrongDeviceName := "wrong-device"
|
||||
@@ -934,8 +934,145 @@ var _ = Describe("DRA Client operations", func() {
|
||||
// Execute
|
||||
resourceMap := make(map[string]*types.ResourceInfo)
|
||||
err = draClient.GetPodResourceMap(pod, resourceMap)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("not found for claim resource"))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(resourceMap).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("should preserve existing kubelet map entries when the claim maps nothing", func() {
|
||||
claimName := "test-claim"
|
||||
deviceName := "device-1"
|
||||
wrongDeviceName := "wrong-device"
|
||||
driverName := "test-driver.example.com"
|
||||
poolName := "test-pool"
|
||||
requestName := "gpu"
|
||||
deviceID := "pci:0000:00:01.0"
|
||||
legacyKey := "example.com/legacy-vf"
|
||||
legacyPCI := "0000:8d:00.4"
|
||||
|
||||
deviceIDValue := deviceID
|
||||
resourceSlice := &resourcev1api.ResourceSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "test-resource-slice"},
|
||||
Spec: resourcev1api.ResourceSliceSpec{
|
||||
Driver: driverName,
|
||||
Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1},
|
||||
Devices: []resourcev1api.Device{
|
||||
{
|
||||
Name: deviceName,
|
||||
Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{
|
||||
multusDeviceIDAttr: {StringValue: &deviceIDValue},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
resourceClaim := &resourcev1api.ResourceClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: claimName, Namespace: "default"},
|
||||
Status: resourcev1api.ResourceClaimStatus{
|
||||
Allocation: &resourcev1api.AllocationResult{
|
||||
Devices: resourcev1api.DeviceAllocationResult{
|
||||
Results: []resourcev1api.DeviceRequestAllocationResult{
|
||||
{Request: requestName, Driver: driverName, Pool: poolName, Device: wrongDeviceName},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
claimNamePtr := claimName
|
||||
pod := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "hybrid-pod", Namespace: "default", UID: k8sTypes.UID("uid-hybrid")},
|
||||
Status: v1.PodStatus{
|
||||
ResourceClaimStatuses: []v1.PodResourceClaimStatus{
|
||||
{Name: claimName, ResourceClaimName: &claimNamePtr},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
resourceMap := map[string]*types.ResourceInfo{
|
||||
legacyKey: {DeviceIDs: []string{legacyPCI}},
|
||||
}
|
||||
err = draClient.GetPodResourceMap(pod, resourceMap)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(resourceMap[legacyKey].DeviceIDs).To(Equal([]string{legacyPCI}))
|
||||
})
|
||||
|
||||
It("should succeed when one allocation result is missing from slices but another resolves", func() {
|
||||
claimName := "multi-claim"
|
||||
driverSRIOV := "sriovnetwork.k8snetworkplumbingwg.io"
|
||||
driverGPU := "gpu.nvidia.com"
|
||||
poolName := "orch-dev-a100-002"
|
||||
deviceVF := "0000-8d-00-4"
|
||||
deviceGPU := "gpu-4"
|
||||
deviceIDVF := "pci:0000:8d:00.4"
|
||||
mapKey := "nvidia.com/port2"
|
||||
deviceIDVFVal := deviceIDVF
|
||||
mapKeyVal := mapKey
|
||||
|
||||
// SR-IOV slice has the VF; empty GPU slice exists so List() finds gpu.nvidia.com/pool (real clusters
|
||||
// always publish a slice per driver/pool). Allocation still references gpu-4, which is not listed here.
|
||||
resourceSlice := &resourcev1api.ResourceSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "sriov-slice"},
|
||||
Spec: resourcev1api.ResourceSliceSpec{
|
||||
Driver: driverSRIOV,
|
||||
Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1},
|
||||
Devices: []resourcev1api.Device{
|
||||
{
|
||||
Name: deviceVF,
|
||||
Attributes: map[resourcev1api.QualifiedName]resourcev1api.DeviceAttribute{
|
||||
multusDeviceIDAttr: {StringValue: &deviceIDVFVal},
|
||||
multusResourceNameAttr: {StringValue: &mapKeyVal},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
gpuSlice := &resourcev1api.ResourceSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "gpu-slice"},
|
||||
Spec: resourcev1api.ResourceSliceSpec{
|
||||
Driver: driverGPU,
|
||||
Pool: resourcev1api.ResourcePool{Name: poolName, ResourceSliceCount: 1},
|
||||
Devices: []resourcev1api.Device{},
|
||||
},
|
||||
}
|
||||
|
||||
resourceClaim := &resourcev1api.ResourceClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: claimName, Namespace: "default"},
|
||||
Status: resourcev1api.ResourceClaimStatus{
|
||||
Allocation: &resourcev1api.AllocationResult{
|
||||
Devices: resourcev1api.DeviceAllocationResult{
|
||||
Results: []resourcev1api.DeviceRequestAllocationResult{
|
||||
{Request: "vf", Driver: driverSRIOV, Pool: poolName, Device: deviceVF},
|
||||
{Request: "gpu", Driver: driverGPU, Pool: poolName, Device: deviceGPU},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
claimNamePtr := claimName
|
||||
pod := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "mixed-pod", Namespace: "default", UID: k8sTypes.UID("uid-mixed")},
|
||||
Status: v1.PodStatus{
|
||||
ResourceClaimStatuses: []v1.PodResourceClaimStatus{
|
||||
{Name: claimName, ResourceClaimName: &claimNamePtr},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := fakeClient.ResourceV1().ResourceClaims("default").Create(context.TODO(), resourceClaim, metav1.CreateOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), resourceSlice, metav1.CreateOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = fakeClient.ResourceV1().ResourceSlices().Create(context.TODO(), gpuSlice, metav1.CreateOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
resourceMap := make(map[string]*types.ResourceInfo)
|
||||
err = draClient.GetPodResourceMap(pod, resourceMap)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(resourceMap[mapKey].DeviceIDs).To(Equal([]string{deviceIDVF}))
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user