Merge pull request #116332 from klueska/extend-resourceclaimstatus

Update resource.AllocationResult with a slice of ResourceHandlers

api/openapi-spec/swagger.json

@@ -13118,15 +13118,19 @@
       "x-kubernetes-map-type": "atomic"
     },
     "io.k8s.api.resource.v1alpha2.AllocationResult": {
-      "description": "AllocationResult contains attributed of an allocated resource.",
+      "description": "AllocationResult contains attributes of an allocated resource.",
       "properties": {
         "availableOnNodes": {
           "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelector",
-          "description": "This field will get set by the resource driver after it has allocated the resource driver to inform the scheduler where it can schedule Pods using the ResourceClaim.\n\nSetting this field is optional. If null, the resource is available everywhere."
+          "description": "This field will get set by the resource driver after it has allocated the resource to inform the scheduler where it can schedule Pods using the ResourceClaim.\n\nSetting this field is optional. If null, the resource is available everywhere."
         },
-        "resourceHandle": {
+        "resourceHandles": {
-          "description": "ResourceHandle contains arbitrary data returned by the driver after a successful allocation. This is opaque for Kubernetes. Driver documentation may explain to users how to interpret this data if needed.\n\nThe maximum size of this field is 16KiB. This may get increased in the future, but not reduced.",
+          "description": "ResourceHandles contain the state associated with an allocation that should be maintained throughout the lifetime of a claim. Each ResourceHandle contains data that should be passed to a specific kubelet plugin once it lands on a node. This data is returned by the driver after a successful allocation and is opaque to Kubernetes. Driver documentation may explain to users how to interpret this data if needed.\n\nSetting this field is optional. It has a maximum size of 32 entries. If null (or empty), it is assumed this allocation will be processed by a single kubelet plugin with no ResourceHandle data attached. The name of the kubelet plugin invoked will match the DriverName set in the ResourceClaimStatus this AllocationResult is embedded in.",
-          "type": "string"
+          "items": {
+            "$ref": "#/definitions/io.k8s.api.resource.v1alpha2.ResourceHandle"
+          },
+          "type": "array",
+          "x-kubernetes-list-type": "atomic"
         },
         "shareable": {
           "description": "Shareable determines whether the resource supports more than one consumer at a time.",
@@ -13405,7 +13409,7 @@
       "properties": {
         "allocation": {
           "$ref": "#/definitions/io.k8s.api.resource.v1alpha2.AllocationResult",
-          "description": "Allocation is set by the resource driver once a resource has been allocated successfully. If this is not specified, the resource is not yet allocated."
+          "description": "Allocation is set by the resource driver once a resource or set of resources has been allocated successfully. If this is not specified, the resources have not been allocated yet."
         },
         "deallocationRequested": {
           "description": "DeallocationRequested indicates that a ResourceClaim is to be deallocated.\n\nThe driver then must deallocate this claim and reset the field together with clearing the Allocation field.\n\nWhile DeallocationRequested is set, no new consumers may be added to ReservedFor.",
@@ -13614,6 +13618,20 @@
       ],
       "type": "object"
     },
+    "io.k8s.api.resource.v1alpha2.ResourceHandle": {
+      "description": "ResourceHandle holds opaque resource data for processing by a specific kubelet plugin.",
+      "properties": {
+        "data": {
+          "description": "Data contains the opaque data associated with this ResourceHandle. It is set by the controller component of the resource driver whose name matches the DriverName set in the ResourceClaimStatus this ResourceHandle is embedded in. It is set at allocation time and is intended for processing by the kubelet plugin whose name matches the DriverName set in this ResourceHandle.\n\nThe maximum size of this field is 16KiB. This may get increased in the future, but not reduced.",
+          "type": "string"
+        },
+        "driverName": {
+          "description": "DriverName specifies the name of the resource driver whose kubelet plugin should be invoked to process this ResourceHandle's data once it lands on a node. This may differ from the DriverName set in ResourceClaimStatus this ResourceHandle is embedded in.",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
     "io.k8s.api.scheduling.v1.PriorityClass": {
       "description": "PriorityClass defines mapping from a priority class name to the priority integer value. The value can be any valid integer.",
       "properties": {

api/openapi-spec/v3/apis__resource.k8s.io__v1alpha2_openapi.json

@@ -83,7 +83,7 @@
         "x-kubernetes-map-type": "atomic"
       },
       "io.k8s.api.resource.v1alpha2.AllocationResult": {
-        "description": "AllocationResult contains attributed of an allocated resource.",
+        "description": "AllocationResult contains attributes of an allocated resource.",
         "properties": {
           "availableOnNodes": {
             "allOf": [
@@ -91,11 +91,20 @@
                 "$ref": "#/components/schemas/io.k8s.api.core.v1.NodeSelector"
               }
             ],
-            "description": "This field will get set by the resource driver after it has allocated the resource driver to inform the scheduler where it can schedule Pods using the ResourceClaim.\n\nSetting this field is optional. If null, the resource is available everywhere."
+            "description": "This field will get set by the resource driver after it has allocated the resource to inform the scheduler where it can schedule Pods using the ResourceClaim.\n\nSetting this field is optional. If null, the resource is available everywhere."
           },
-          "resourceHandle": {
+          "resourceHandles": {
-            "description": "ResourceHandle contains arbitrary data returned by the driver after a successful allocation. This is opaque for Kubernetes. Driver documentation may explain to users how to interpret this data if needed.\n\nThe maximum size of this field is 16KiB. This may get increased in the future, but not reduced.",
+            "description": "ResourceHandles contain the state associated with an allocation that should be maintained throughout the lifetime of a claim. Each ResourceHandle contains data that should be passed to a specific kubelet plugin once it lands on a node. This data is returned by the driver after a successful allocation and is opaque to Kubernetes. Driver documentation may explain to users how to interpret this data if needed.\n\nSetting this field is optional. It has a maximum size of 32 entries. If null (or empty), it is assumed this allocation will be processed by a single kubelet plugin with no ResourceHandle data attached. The name of the kubelet plugin invoked will match the DriverName set in the ResourceClaimStatus this AllocationResult is embedded in.",
-            "type": "string"
+            "items": {
+              "allOf": [
+                {
+                  "$ref": "#/components/schemas/io.k8s.api.resource.v1alpha2.ResourceHandle"
+                }
+              ],
+              "default": {}
+            },
+            "type": "array",
+            "x-kubernetes-list-type": "atomic"
           },
           "shareable": {
             "description": "Shareable determines whether the resource supports more than one consumer at a time.",
@@ -445,7 +454,7 @@
                 "$ref": "#/components/schemas/io.k8s.api.resource.v1alpha2.AllocationResult"
               }
             ],
-            "description": "Allocation is set by the resource driver once a resource has been allocated successfully. If this is not specified, the resource is not yet allocated."
+            "description": "Allocation is set by the resource driver once a resource or set of resources has been allocated successfully. If this is not specified, the resources have not been allocated yet."
           },
           "deallocationRequested": {
             "description": "DeallocationRequested indicates that a ResourceClaim is to be deallocated.\n\nThe driver then must deallocate this claim and reset the field together with clearing the Allocation field.\n\nWhile DeallocationRequested is set, no new consumers may be added to ReservedFor.",
@@ -715,6 +724,20 @@
         ],
         "type": "object"
       },
+      "io.k8s.api.resource.v1alpha2.ResourceHandle": {
+        "description": "ResourceHandle holds opaque resource data for processing by a specific kubelet plugin.",
+        "properties": {
+          "data": {
+            "description": "Data contains the opaque data associated with this ResourceHandle. It is set by the controller component of the resource driver whose name matches the DriverName set in the ResourceClaimStatus this ResourceHandle is embedded in. It is set at allocation time and is intended for processing by the kubelet plugin whose name matches the DriverName set in this ResourceHandle.\n\nThe maximum size of this field is 16KiB. This may get increased in the future, but not reduced.",
+            "type": "string"
+          },
+          "driverName": {
+            "description": "DriverName specifies the name of the resource driver whose kubelet plugin should be invoked to process this ResourceHandle's data once it lands on a node. This may differ from the DriverName set in ResourceClaimStatus this ResourceHandle is embedded in.",
+            "type": "string"
+          }
+        },
+        "type": "object"
+      },
       "io.k8s.apimachinery.pkg.apis.meta.v1.APIResource": {
         "description": "APIResource specifies the name of a resource and whether it is namespaced.",
         "properties": {

pkg/apis/resource/types.go

@@ -97,9 +97,9 @@ type ResourceClaimStatus struct {
 	// +optional
 	DriverName string
 
-	// Allocation is set by the resource driver once a resource has been
+	// Allocation is set by the resource driver once a resource or set of
-	// allocated successfully. If this is not specified, the resource is
+	// resources has been allocated successfully. If this is not specified, the
-	// not yet allocated.
+	// resources have not been allocated yet.
 	// +optional
 	Allocation *AllocationResult
 
@@ -128,21 +128,28 @@ type ResourceClaimStatus struct {
 // claim.status.reservedFor.
 const ResourceClaimReservedForMaxSize = 32
 
-// AllocationResult contains attributed of an allocated resource.
+// AllocationResult contains attributes of an allocated resource.
 type AllocationResult struct {
-	// ResourceHandle contains arbitrary data returned by the driver after a
+	// ResourceHandles contain the state associated with an allocation that
-	// successful allocation. This is opaque for
+	// should be maintained throughout the lifetime of a claim. Each
-	// Kubernetes. Driver documentation may explain to users how to
+	// ResourceHandle contains data that should be passed to a specific kubelet
-	// interpret this data if needed.
+	// plugin once it lands on a node. This data is returned by the driver
+	// after a successful allocation and is opaque to Kubernetes. Driver
+	// documentation may explain to users how to interpret this data if needed.
 	//
-	// The maximum size of this field is 16KiB. This may get
+	// Setting this field is optional. It has a maximum size of 32 entries.
-	// increased in the future, but not reduced.
+	// If null (or empty), it is assumed this allocation will be processed by a
+	// single kubelet plugin with no ResourceHandle data attached. The name of
+	// the kubelet plugin invoked will match the DriverName set in the
+	// ResourceClaimStatus this AllocationResult is embedded in.
+	//
+	// +listType=atomic
 	// +optional
-	ResourceHandle string
+	ResourceHandles []ResourceHandle
 
-	// This field will get set by the resource driver after it has
+	// This field will get set by the resource driver after it has allocated
-	// allocated the resource driver to inform the scheduler where it can
+	// the resource to inform the scheduler where it can schedule Pods using
-	// schedule Pods using the ResourceClaim.
+	// the ResourceClaim.
 	//
 	// Setting this field is optional. If null, the resource is available
 	// everywhere.
@@ -155,8 +162,33 @@ type AllocationResult struct {
 	Shareable bool
 }
 
-// ResourceHandleMaxSize is the maximum size of allocation.resourceHandle.
+// AllocationResultResourceHandlesMaxSize represents the maximum number of
-const ResourceHandleMaxSize = 16 * 1024
+// entries in allocation.resourceHandles.
+const AllocationResultResourceHandlesMaxSize = 32
+
+// ResourceHandle holds opaque resource data for processing by a specific kubelet plugin.
+type ResourceHandle struct {
+	// DriverName specifies the name of the resource driver whose kubelet
+	// plugin should be invoked to process this ResourceHandle's data once it
+	// lands on a node. This may differ from the DriverName set in
+	// ResourceClaimStatus this ResourceHandle is embedded in.
+	DriverName string
+
+	// Data contains the opaque data associated with this ResourceHandle. It is
+	// set by the controller component of the resource driver whose name
+	// matches the DriverName set in the ResourceClaimStatus this
+	// ResourceHandle is embedded in. It is set at allocation time and is
+	// intended for processing by the kubelet plugin whose name matches
+	// the DriverName set in this ResourceHandle.
+	//
+	// The maximum size of this field is 16KiB. This may get increased in the
+	// future, but not reduced.
+	// +optional
+	Data string
+}
+
+// ResourceHandleDataMaxSize represents the maximum size of resourceHandle.data.
+const ResourceHandleDataMaxSize = 16 * 1024
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 

pkg/apis/resource/v1alpha2/zz_generated.conversion.go

@@ -220,11 +220,21 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*v1alpha2.ResourceHandle)(nil), (*resource.ResourceHandle)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_ResourceHandle_To_resource_ResourceHandle(a.(*v1alpha2.ResourceHandle), b.(*resource.ResourceHandle), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*resource.ResourceHandle)(nil), (*v1alpha2.ResourceHandle)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_resource_ResourceHandle_To_v1alpha2_ResourceHandle(a.(*resource.ResourceHandle), b.(*v1alpha2.ResourceHandle), scope)
+	}); err != nil {
+		return err
+	}
 	return nil
 }
 
 func autoConvert_v1alpha2_AllocationResult_To_resource_AllocationResult(in *v1alpha2.AllocationResult, out *resource.AllocationResult, s conversion.Scope) error {
-	out.ResourceHandle = in.ResourceHandle
+	out.ResourceHandles = *(*[]resource.ResourceHandle)(unsafe.Pointer(&in.ResourceHandles))
 	out.AvailableOnNodes = (*core.NodeSelector)(unsafe.Pointer(in.AvailableOnNodes))
 	out.Shareable = in.Shareable
 	return nil
@@ -236,7 +246,7 @@ func Convert_v1alpha2_AllocationResult_To_resource_AllocationResult(in *v1alpha2
 }
 
 func autoConvert_resource_AllocationResult_To_v1alpha2_AllocationResult(in *resource.AllocationResult, out *v1alpha2.AllocationResult, s conversion.Scope) error {
-	out.ResourceHandle = in.ResourceHandle
+	out.ResourceHandles = *(*[]v1alpha2.ResourceHandle)(unsafe.Pointer(&in.ResourceHandles))
 	out.AvailableOnNodes = (*v1.NodeSelector)(unsafe.Pointer(in.AvailableOnNodes))
 	out.Shareable = in.Shareable
 	return nil
@@ -666,3 +676,25 @@ func autoConvert_resource_ResourceClassParametersReference_To_v1alpha2_ResourceC
 func Convert_resource_ResourceClassParametersReference_To_v1alpha2_ResourceClassParametersReference(in *resource.ResourceClassParametersReference, out *v1alpha2.ResourceClassParametersReference, s conversion.Scope) error {
 	return autoConvert_resource_ResourceClassParametersReference_To_v1alpha2_ResourceClassParametersReference(in, out, s)
 }
+
+func autoConvert_v1alpha2_ResourceHandle_To_resource_ResourceHandle(in *v1alpha2.ResourceHandle, out *resource.ResourceHandle, s conversion.Scope) error {
+	out.DriverName = in.DriverName
+	out.Data = in.Data
+	return nil
+}
+
+// Convert_v1alpha2_ResourceHandle_To_resource_ResourceHandle is an autogenerated conversion function.
+func Convert_v1alpha2_ResourceHandle_To_resource_ResourceHandle(in *v1alpha2.ResourceHandle, out *resource.ResourceHandle, s conversion.Scope) error {
+	return autoConvert_v1alpha2_ResourceHandle_To_resource_ResourceHandle(in, out, s)
+}
+
+func autoConvert_resource_ResourceHandle_To_v1alpha2_ResourceHandle(in *resource.ResourceHandle, out *v1alpha2.ResourceHandle, s conversion.Scope) error {
+	out.DriverName = in.DriverName
+	out.Data = in.Data
+	return nil
+}
+
+// Convert_resource_ResourceHandle_To_v1alpha2_ResourceHandle is an autogenerated conversion function.
+func Convert_resource_ResourceHandle_To_v1alpha2_ResourceHandle(in *resource.ResourceHandle, out *v1alpha2.ResourceHandle, s conversion.Scope) error {
+	return autoConvert_resource_ResourceHandle_To_v1alpha2_ResourceHandle(in, out, s)
+}

pkg/apis/resource/validation/validation.go

@@ -147,6 +147,11 @@ func ValidateClaimStatusUpdate(resourceClaim, oldClaim *resource.ResourceClaim)
 		}
 	}
 
+	// Updates to a populated resourceClaim.Status.Allocation are not allowed
+	if oldClaim.Status.Allocation != nil && resourceClaim.Status.Allocation != nil {
+		allErrs = append(allErrs, apimachineryvalidation.ValidateImmutableField(resourceClaim.Status.Allocation, oldClaim.Status.Allocation, fldPath.Child("allocation"))...)
+	}
+
 	if !oldClaim.Status.DeallocationRequested &&
 		resourceClaim.Status.DeallocationRequested &&
 		len(resourceClaim.Status.ReservedFor) > 0 {
@@ -177,8 +182,8 @@ func ValidateClaimStatusUpdate(resourceClaim, oldClaim *resource.ResourceClaim)
 func validateAllocationResult(allocation *resource.AllocationResult, fldPath *field.Path) field.ErrorList {
 	var allErrs field.ErrorList
 	if allocation != nil {
-		if len(allocation.ResourceHandle) > resource.ResourceHandleMaxSize {
+		if len(allocation.ResourceHandles) > 0 {
-			allErrs = append(allErrs, field.TooLongMaxLength(fldPath.Child("resourceHandle"), len(allocation.ResourceHandle), resource.ResourceHandleMaxSize))
+			allErrs = append(allErrs, validateResourceHandles(allocation.ResourceHandles, resource.AllocationResultResourceHandlesMaxSize, fldPath.Child("resourceHandles"))...)
 		}
 		if allocation.AvailableOnNodes != nil {
 			allErrs = append(allErrs, corevalidation.ValidateNodeSelector(allocation.AvailableOnNodes, fldPath.Child("availableOnNodes"))...)
@@ -187,6 +192,24 @@ func validateAllocationResult(allocation *resource.AllocationResult, fldPath *fi
 	return allErrs
 }
 
+func validateResourceHandles(resourceHandles []resource.ResourceHandle, maxSize int, fldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+	for i, resourceHandle := range resourceHandles {
+		idxPath := fldPath.Index(i)
+		allErrs = append(allErrs, validateResourceDriverName(resourceHandle.DriverName, idxPath.Child("driverName"))...)
+		if len(resourceHandle.Data) > resource.ResourceHandleDataMaxSize {
+			allErrs = append(allErrs, field.TooLongMaxLength(idxPath.Child("data"), len(resourceHandle.Data), resource.ResourceHandleDataMaxSize))
+		}
+	}
+	if len(resourceHandles) > maxSize {
+		// Dumping the entire field into the error message is likely to be too long,
+		// in particular when it is already beyond the maximum size. Instead this
+		// just shows the number of entries.
+		allErrs = append(allErrs, field.TooLongMaxLength(fldPath, len(resourceHandles), maxSize))
+	}
+	return allErrs
+}
+
 func validateResourceClaimUserReference(ref resource.ResourceClaimConsumerReference, fldPath *field.Path) field.ErrorList {
 	var allErrs field.ErrorList
 	if ref.Resource == "" {

pkg/apis/resource/validation/validation_resourceclaim_test.go

@@ -315,6 +315,7 @@ func TestValidateClaimUpdate(t *testing.T) {
 }
 
 func TestValidateClaimStatusUpdate(t *testing.T) {
+	invalidName := "!@#$%^"
 	validClaim := testClaim("foo", "ns", resource.ResourceClaimSpec{
 		ResourceClassName: "valid",
 		AllocationMode:    resource.AllocationModeImmediate,
@@ -324,8 +325,18 @@ func TestValidateClaimStatusUpdate(t *testing.T) {
 	validAllocatedClaim.Status = resource.ResourceClaimStatus{
 		DriverName: "valid",
 		Allocation: &resource.AllocationResult{
-			ResourceHandle: strings.Repeat(" ", resource.ResourceHandleMaxSize),
+			ResourceHandles: func() []resource.ResourceHandle {
-			Shareable:      true,
+				var handles []resource.ResourceHandle
+				for i := 0; i < resource.AllocationResultResourceHandlesMaxSize; i++ {
+					handle := resource.ResourceHandle{
+						DriverName: "valid",
+						Data:       strings.Repeat(" ", resource.ResourceHandleDataMaxSize),
+					}
+					handles = append(handles, handle)
+				}
+				return handles
+			}(),
+			Shareable: true,
 		},
 	}
 
@@ -359,18 +370,60 @@ func TestValidateClaimStatusUpdate(t *testing.T) {
 			update: func(claim *resource.ResourceClaim) *resource.ResourceClaim {
 				claim.Status.DriverName = "valid"
 				claim.Status.Allocation = &resource.AllocationResult{
-					ResourceHandle: strings.Repeat(" ", resource.ResourceHandleMaxSize),
+					ResourceHandles: []resource.ResourceHandle{
+						{
+							DriverName: "valid",
+							Data:       strings.Repeat(" ", resource.ResourceHandleDataMaxSize),
+						},
+					},
 				}
 				return claim
 			},
 		},
-		"invalid-allocation-handle": {
+		"invalid-allocation-resourceHandles": {
-			wantFailures: field.ErrorList{field.TooLongMaxLength(field.NewPath("status", "allocation", "resourceHandle"), resource.ResourceHandleMaxSize+1, resource.ResourceHandleMaxSize)},
+			wantFailures: field.ErrorList{field.TooLongMaxLength(field.NewPath("status", "allocation", "resourceHandles"), resource.AllocationResultResourceHandlesMaxSize+1, resource.AllocationResultResourceHandlesMaxSize)},
 			oldClaim:     validClaim,
 			update: func(claim *resource.ResourceClaim) *resource.ResourceClaim {
 				claim.Status.DriverName = "valid"
 				claim.Status.Allocation = &resource.AllocationResult{
-					ResourceHandle: strings.Repeat(" ", resource.ResourceHandleMaxSize+1),
+					ResourceHandles: func() []resource.ResourceHandle {
+						var handles []resource.ResourceHandle
+						for i := 0; i < resource.AllocationResultResourceHandlesMaxSize+1; i++ {
+							handles = append(handles, resource.ResourceHandle{DriverName: "valid"})
+						}
+						return handles
+					}(),
+				}
+				return claim
+			},
+		},
+		"invalid-allocation-resource-handle-drivername": {
+			wantFailures: field.ErrorList{field.Invalid(field.NewPath("status", "allocation", "resourceHandles[0]", "driverName"), invalidName, "a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')")},
+			oldClaim:     validClaim,
+			update: func(claim *resource.ResourceClaim) *resource.ResourceClaim {
+				claim.Status.DriverName = "valid"
+				claim.Status.Allocation = &resource.AllocationResult{
+					ResourceHandles: []resource.ResourceHandle{
+						{
+							DriverName: invalidName,
+						},
+					},
+				}
+				return claim
+			},
+		},
+		"invalid-allocation-resource-handle-data": {
+			wantFailures: field.ErrorList{field.TooLongMaxLength(field.NewPath("status", "allocation", "resourceHandles[0]", "data"), resource.ResourceHandleDataMaxSize+1, resource.ResourceHandleDataMaxSize)},
+			oldClaim:     validClaim,
+			update: func(claim *resource.ResourceClaim) *resource.ResourceClaim {
+				claim.Status.DriverName = "valid"
+				claim.Status.Allocation = &resource.AllocationResult{
+					ResourceHandles: []resource.ResourceHandle{
+						{
+							DriverName: "valid",
+							Data:       strings.Repeat(" ", resource.ResourceHandleDataMaxSize+1),
+						},
+					},
 				}
 				return claim
 			},
@@ -564,6 +617,18 @@ func TestValidateClaimStatusUpdate(t *testing.T) {
 				return claim
 			},
 		},
+		"remove-allocation": {
+			oldClaim: func() *resource.ResourceClaim {
+				claim := validAllocatedClaim.DeepCopy()
+				claim.Status.DeallocationRequested = true
+				return claim
+			}(),
+			update: func(claim *resource.ResourceClaim) *resource.ResourceClaim {
+				claim.Status.DeallocationRequested = false
+				claim.Status.Allocation = nil
+				return claim
+			},
+		},
 		"invalid-deallocation-requested-removal": {
 			wantFailures: field.ErrorList{field.Forbidden(field.NewPath("status", "deallocationRequested"), "may not be cleared when `allocation` is set")},
 			oldClaim: func() *resource.ResourceClaim {
@@ -576,6 +641,32 @@ func TestValidateClaimStatusUpdate(t *testing.T) {
 				return claim
 			},
 		},
+		"invalid-allocation-modification": {
+			wantFailures: field.ErrorList{field.Invalid(field.NewPath("status.allocation"), func() *resource.AllocationResult {
+				claim := validAllocatedClaim.DeepCopy()
+				claim.Status.Allocation.ResourceHandles = []resource.ResourceHandle{
+					{
+						DriverName: "valid",
+						Data:       strings.Repeat(" ", resource.ResourceHandleDataMaxSize/2),
+					},
+				}
+				return claim.Status.Allocation
+			}(), "field is immutable")},
+			oldClaim: func() *resource.ResourceClaim {
+				claim := validAllocatedClaim.DeepCopy()
+				claim.Status.DeallocationRequested = false
+				return claim
+			}(),
+			update: func(claim *resource.ResourceClaim) *resource.ResourceClaim {
+				claim.Status.Allocation.ResourceHandles = []resource.ResourceHandle{
+					{
+						DriverName: "valid",
+						Data:       strings.Repeat(" ", resource.ResourceHandleDataMaxSize/2),
+					},
+				}
+				return claim
+			},
+		},
 		"invalid-deallocation-requested-in-use": {
 			wantFailures: field.ErrorList{field.Forbidden(field.NewPath("status", "deallocationRequested"), "deallocation cannot be requested while `reservedFor` is set")},
 			oldClaim: func() *resource.ResourceClaim {

pkg/apis/resource/zz_generated.deepcopy.go

@@ -29,6 +29,11 @@ import (
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AllocationResult) DeepCopyInto(out *AllocationResult) {
 	*out = *in
+	if in.ResourceHandles != nil {
+		in, out := &in.ResourceHandles, &out.ResourceHandles
+		*out = make([]ResourceHandle, len(*in))
+		copy(*out, *in)
+	}
 	if in.AvailableOnNodes != nil {
 		in, out := &in.AvailableOnNodes, &out.AvailableOnNodes
 		*out = new(core.NodeSelector)
@@ -475,3 +480,19 @@ func (in *ResourceClassParametersReference) DeepCopy() *ResourceClassParametersR
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ResourceHandle) DeepCopyInto(out *ResourceHandle) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceHandle.
+func (in *ResourceHandle) DeepCopy() *ResourceHandle {
+	if in == nil {
+		return nil
+	}
+	out := new(ResourceHandle)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/generated/openapi/zz_generated.openapi.go

@@ -846,6 +846,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 		"k8s.io/api/resource/v1alpha2.ResourceClass":                                                      schema_k8sio_api_resource_v1alpha2_ResourceClass(ref),
 		"k8s.io/api/resource/v1alpha2.ResourceClassList":                                                  schema_k8sio_api_resource_v1alpha2_ResourceClassList(ref),
 		"k8s.io/api/resource/v1alpha2.ResourceClassParametersReference":                                   schema_k8sio_api_resource_v1alpha2_ResourceClassParametersReference(ref),
+		"k8s.io/api/resource/v1alpha2.ResourceHandle":                                                     schema_k8sio_api_resource_v1alpha2_ResourceHandle(ref),
 		"k8s.io/api/scheduling/v1.PriorityClass":                                                          schema_k8sio_api_scheduling_v1_PriorityClass(ref),
 		"k8s.io/api/scheduling/v1.PriorityClassList":                                                      schema_k8sio_api_scheduling_v1_PriorityClassList(ref),
 		"k8s.io/api/scheduling/v1alpha1.PriorityClass":                                                    schema_k8sio_api_scheduling_v1alpha1_PriorityClass(ref),
@@ -41292,19 +41293,31 @@ func schema_k8sio_api_resource_v1alpha2_AllocationResult(ref common.ReferenceCal
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
 			SchemaProps: spec.SchemaProps{
-				Description: "AllocationResult contains attributed of an allocated resource.",
+				Description: "AllocationResult contains attributes of an allocated resource.",
 				Type:        []string{"object"},
 				Properties: map[string]spec.Schema{
-					"resourceHandle": {
+					"resourceHandles": {
+						VendorExtensible: spec.VendorExtensible{
+							Extensions: spec.Extensions{
+								"x-kubernetes-list-type": "atomic",
+							},
+						},
 						SchemaProps: spec.SchemaProps{
-							Description: "ResourceHandle contains arbitrary data returned by the driver after a successful allocation. This is opaque for Kubernetes. Driver documentation may explain to users how to interpret this data if needed.\n\nThe maximum size of this field is 16KiB. This may get increased in the future, but not reduced.",
+							Description: "ResourceHandles contain the state associated with an allocation that should be maintained throughout the lifetime of a claim. Each ResourceHandle contains data that should be passed to a specific kubelet plugin once it lands on a node. This data is returned by the driver after a successful allocation and is opaque to Kubernetes. Driver documentation may explain to users how to interpret this data if needed.\n\nSetting this field is optional. It has a maximum size of 32 entries. If null (or empty), it is assumed this allocation will be processed by a single kubelet plugin with no ResourceHandle data attached. The name of the kubelet plugin invoked will match the DriverName set in the ResourceClaimStatus this AllocationResult is embedded in.",
-							Type:        []string{"string"},
+							Type:        []string{"array"},
-							Format:      "",
+							Items: &spec.SchemaOrArray{
+								Schema: &spec.Schema{
+									SchemaProps: spec.SchemaProps{
+										Default: map[string]interface{}{},
+										Ref:     ref("k8s.io/api/resource/v1alpha2.ResourceHandle"),
+									},
+								},
+							},
 						},
 					},
 					"availableOnNodes": {
 						SchemaProps: spec.SchemaProps{
-							Description: "This field will get set by the resource driver after it has allocated the resource driver to inform the scheduler where it can schedule Pods using the ResourceClaim.\n\nSetting this field is optional. If null, the resource is available everywhere.",
+							Description: "This field will get set by the resource driver after it has allocated the resource to inform the scheduler where it can schedule Pods using the ResourceClaim.\n\nSetting this field is optional. If null, the resource is available everywhere.",
 							Ref:         ref("k8s.io/api/core/v1.NodeSelector"),
 						},
 					},
@@ -41319,7 +41332,7 @@ func schema_k8sio_api_resource_v1alpha2_AllocationResult(ref common.ReferenceCal
 			},
 		},
 		Dependencies: []string{
-			"k8s.io/api/core/v1.NodeSelector"},
+			"k8s.io/api/core/v1.NodeSelector", "k8s.io/api/resource/v1alpha2.ResourceHandle"},
 	}
 }
 
@@ -41779,7 +41792,7 @@ func schema_k8sio_api_resource_v1alpha2_ResourceClaimStatus(ref common.Reference
 					},
 					"allocation": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Allocation is set by the resource driver once a resource has been allocated successfully. If this is not specified, the resource is not yet allocated.",
+							Description: "Allocation is set by the resource driver once a resource or set of resources has been allocated successfully. If this is not specified, the resources have not been allocated yet.",
 							Ref:         ref("k8s.io/api/resource/v1alpha2.AllocationResult"),
 						},
 					},
@@ -42097,6 +42110,33 @@ func schema_k8sio_api_resource_v1alpha2_ResourceClassParametersReference(ref com
 	}
 }
 
+func schema_k8sio_api_resource_v1alpha2_ResourceHandle(ref common.ReferenceCallback) common.OpenAPIDefinition {
+	return common.OpenAPIDefinition{
+		Schema: spec.Schema{
+			SchemaProps: spec.SchemaProps{
+				Description: "ResourceHandle holds opaque resource data for processing by a specific kubelet plugin.",
+				Type:        []string{"object"},
+				Properties: map[string]spec.Schema{
+					"driverName": {
+						SchemaProps: spec.SchemaProps{
+							Description: "DriverName specifies the name of the resource driver whose kubelet plugin should be invoked to process this ResourceHandle's data once it lands on a node. This may differ from the DriverName set in ResourceClaimStatus this ResourceHandle is embedded in.",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+					"data": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Data contains the opaque data associated with this ResourceHandle. It is set by the controller component of the resource driver whose name matches the DriverName set in the ResourceClaimStatus this ResourceHandle is embedded in. It is set at allocation time and is intended for processing by the kubelet plugin whose name matches the DriverName set in this ResourceHandle.\n\nThe maximum size of this field is 16KiB. This may get increased in the future, but not reduced.",
+							Type:        []string{"string"},
+							Format:      "",
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func schema_k8sio_api_scheduling_v1_PriorityClass(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{

pkg/kubelet/cm/dra/manager.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	v1 "k8s.io/api/core/v1"
+	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -101,22 +102,13 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error {
 						pod.Name, pod.UID, podResourceClaim.Name, resourceClaim.UID)
 				}
 
-				// Build a slice of "resourceHandles" to group the name of the
+				// Grab the allocation.resourceHandles. If there are no
-				// kubelet plugin to call NodePrepareResources() on and the
+				// allocation.resourceHandles, create a single resourceHandle with no
-				// ResourceHandle data to be processed by that plugin. For now
+				// content. This will trigger processing of this claim by a single
-				// this slice will only have a single entry, where the name of
+				// kubelet plugin whose name matches resourceClaim.Status.DriverName.
-				// the kubelet plugin matches the DriverName. In the future we
+				resourceHandles := resourceClaim.Status.Allocation.ResourceHandles
-				// plan to allow each claim to be processed by multiple plugins
+				if len(resourceHandles) == 0 {
-				// (each with their own ResourceHandle) so this code is being
+					resourceHandles = make([]resourcev1alpha2.ResourceHandle, 1)
-				// written in a way to accommodate this.
-				resourceHandles := []struct {
-					KubeletPluginName string
-					Data              string
-				}{
-					{
-						KubeletPluginName: resourceClaim.Status.DriverName,
-						Data:              resourceClaim.Status.Allocation.ResourceHandle,
-					},
 				}
 
 				// Create a claimInfo object to store the relevant claim info.
@@ -131,10 +123,17 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error {
 
 				// Walk through each resourceHandle
 				for _, resourceHandle := range resourceHandles {
+					// If no DriverName is provided in the resourceHandle, we
+					// use the DriverName from the status
+					pluginName := resourceHandle.DriverName
+					if pluginName == "" {
+						pluginName = resourceClaim.Status.DriverName
+					}
+
 					// Call NodePrepareResource RPC for each resourceHandle
-					client, err := dra.NewDRAPluginClient(resourceHandle.KubeletPluginName)
+					client, err := dra.NewDRAPluginClient(pluginName)
 					if err != nil {
-						return fmt.Errorf("failed to get DRA Plugin client for plugin name %s, err=%+v", resourceHandle.KubeletPluginName, err)
+						return fmt.Errorf("failed to get DRA Plugin client for plugin name %s, err=%+v", pluginName, err)
 					}
 					response, err := client.NodePrepareResource(
 						context.Background(),
@@ -146,11 +145,11 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error {
 						return fmt.Errorf("NodePrepareResource failed, claim UID: %s, claim name: %s, resource handle: %s, err: %+v",
 							resourceClaim.UID, resourceClaim.Name, resourceHandle.Data, err)
 					}
-					klog.V(3).InfoS("NodePrepareResource succeeded", "pluginName", resourceHandle.KubeletPluginName, "response", response)
+					klog.V(3).InfoS("NodePrepareResource succeeded", "pluginName", pluginName, "response", response)
 
 					// Add the CDI Devices returned by NodePrepareResource to
 					// the claimInfo object.
-					err = claimInfo.addCDIDevices(resourceHandle.KubeletPluginName, response.CdiDevices)
+					err = claimInfo.addCDIDevices(pluginName, response.CdiDevices)
 					if err != nil {
 						return fmt.Errorf("failed to add CDIDevices to claimInfo %+v: %+v", claimInfo, err)
 					}

pkg/kubelet/cm/dra/state/state_checkpoint.go

@@ -54,7 +54,7 @@ type ClaimInfoState struct {
 	// PodUIDs is a set of pod UIDs that reference a resource
 	PodUIDs sets.Set[string]
 
-	// CDIDevices is a map of KubeletPluginName --> CDI devices returned by the
+	// CDIDevices is a map of DriverName --> CDI devices returned by the
 	// GRPC API call NodePrepareResource
 	CDIDevices map[string][]string
 }

staging/src/k8s.io/api/resource/v1alpha2/generated.pb.go

@@ -550,6 +550,34 @@ func (m *ResourceClassParametersReference) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_ResourceClassParametersReference proto.InternalMessageInfo
 
+func (m *ResourceHandle) Reset()      { *m = ResourceHandle{} }
+func (*ResourceHandle) ProtoMessage() {}
+func (*ResourceHandle) Descriptor() ([]byte, []int) {
+	return fileDescriptor_3add37bbd52889e0, []int{18}
+}
+func (m *ResourceHandle) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ResourceHandle) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	b = b[:cap(b)]
+	n, err := m.MarshalToSizedBuffer(b)
+	if err != nil {
+		return nil, err
+	}
+	return b[:n], nil
+}
+func (m *ResourceHandle) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ResourceHandle.Merge(m, src)
+}
+func (m *ResourceHandle) XXX_Size() int {
+	return m.Size()
+}
+func (m *ResourceHandle) XXX_DiscardUnknown() {
+	xxx_messageInfo_ResourceHandle.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ResourceHandle proto.InternalMessageInfo
+
 func init() {
 	proto.RegisterType((*AllocationResult)(nil), "k8s.io.api.resource.v1alpha2.AllocationResult")
 	proto.RegisterType((*PodSchedulingContext)(nil), "k8s.io.api.resource.v1alpha2.PodSchedulingContext")
@@ -569,6 +597,7 @@ func init() {
 	proto.RegisterType((*ResourceClass)(nil), "k8s.io.api.resource.v1alpha2.ResourceClass")
 	proto.RegisterType((*ResourceClassList)(nil), "k8s.io.api.resource.v1alpha2.ResourceClassList")
 	proto.RegisterType((*ResourceClassParametersReference)(nil), "k8s.io.api.resource.v1alpha2.ResourceClassParametersReference")
+	proto.RegisterType((*ResourceHandle)(nil), "k8s.io.api.resource.v1alpha2.ResourceHandle")
 }
 
 func init() {
@@ -576,83 +605,85 @@ func init() {
 }
 
 var fileDescriptor_3add37bbd52889e0 = []byte{
-	// 1209 bytes of a gzipped FileDescriptorProto
+	// 1233 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0xcf, 0x6e, 0xe3, 0xd4,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0x4f, 0x6f, 0x1b, 0x45,
-	0x17, 0xae, 0x93, 0x74, 0xd4, 0xde, 0xb4, 0x99, 0xd6, 0x6d, 0x7f, 0xca, 0x54, 0x33, 0x49, 0x7e,
+	0x14, 0xcf, 0xda, 0x6e, 0x95, 0x4c, 0x1a, 0x37, 0xd9, 0xb6, 0xe0, 0x46, 0xad, 0x63, 0xf6, 0x14,
-	0x5e, 0x45, 0x02, 0xec, 0x69, 0x40, 0x43, 0xc5, 0x9f, 0x91, 0xea, 0x16, 0x86, 0x0a, 0xa6, 0x13,
+	0x89, 0xb2, 0xdb, 0x06, 0x54, 0x2a, 0xfe, 0x49, 0xd9, 0x06, 0x4a, 0x04, 0x4d, 0xc3, 0x98, 0x8a,
-	0x6e, 0xa8, 0x98, 0x22, 0x84, 0xe6, 0xc6, 0x3e, 0x93, 0x98, 0xfa, 0x1f, 0xbe, 0xd7, 0x81, 0x11,
+	0x16, 0x21, 0xd4, 0xc9, 0xee, 0xab, 0xbd, 0x64, 0xff, 0xb1, 0x33, 0x6b, 0xa8, 0xb8, 0xf4, 0x23,
-	0x9b, 0x79, 0x84, 0x59, 0xb0, 0x61, 0xc5, 0x92, 0x17, 0xe0, 0x0d, 0x10, 0x52, 0x97, 0x45, 0xb0,
+	0xf4, 0xc0, 0x01, 0x4e, 0x1c, 0xf9, 0x02, 0x7c, 0x03, 0x84, 0xd4, 0x63, 0x11, 0x1c, 0x7a, 0xb2,
-	0x98, 0x55, 0x44, 0xc3, 0x82, 0x07, 0x60, 0xc5, 0xac, 0x90, 0x1d, 0xdb, 0xb1, 0x9d, 0x38, 0x34,
+	0xa8, 0xf9, 0x08, 0x9c, 0xe8, 0x09, 0xcd, 0x78, 0x77, 0xbd, 0xb3, 0xf6, 0x9a, 0x38, 0x07, 0x0b,
-	0x5d, 0x44, 0xb0, 0x9a, 0xf1, 0x3d, 0xdf, 0xf9, 0xee, 0xb9, 0xdf, 0xb9, 0xe7, 0xdc, 0x93, 0xa2,
+	0x4e, 0xc9, 0xcc, 0xfb, 0xbd, 0xdf, 0xfb, 0x37, 0xef, 0xcd, 0xac, 0xd1, 0xbb, 0x87, 0xd7, 0xa8,
-	0x77, 0x4e, 0x77, 0xa9, 0xa8, 0x59, 0xd2, 0xa9, 0xdb, 0x06, 0xc7, 0x04, 0x06, 0x54, 0xea, 0x81,
+	0xee, 0x04, 0xc6, 0x61, 0x7c, 0x00, 0x91, 0x0f, 0x0c, 0xa8, 0xd1, 0x03, 0xdf, 0x0e, 0x22, 0x23,
-	0xa9, 0x5a, 0x8e, 0x14, 0x18, 0x88, 0xad, 0x49, 0x0e, 0x50, 0xcb, 0x75, 0x14, 0x90, 0x7a, 0x3b,
+	0x11, 0x90, 0xd0, 0x31, 0x22, 0xa0, 0x41, 0x1c, 0x59, 0x60, 0xf4, 0xae, 0x10, 0x37, 0xec, 0x92,
-	0x44, 0xb7, 0xbb, 0xa4, 0x21, 0x75, 0xc0, 0x04, 0x87, 0x30, 0x50, 0x45, 0xdb, 0xb1, 0x98, 0xc5,
+	0x2d, 0xa3, 0x03, 0x3e, 0x44, 0x84, 0x81, 0xad, 0x87, 0x51, 0xc0, 0x02, 0xf5, 0xc2, 0x10, 0xad,
-	0xdf, 0x1c, 0xa2, 0x45, 0x62, 0x6b, 0x62, 0x88, 0x16, 0x43, 0xf4, 0xf6, 0x2b, 0x1d, 0x8d, 0x75,
+	0x93, 0xd0, 0xd1, 0x53, 0xb4, 0x9e, 0xa2, 0xd7, 0x5f, 0xe9, 0x38, 0xac, 0x1b, 0x1f, 0xe8, 0x56,
-	0xdd, 0xb6, 0xa8, 0x58, 0x86, 0xd4, 0xb1, 0x3a, 0x96, 0xe4, 0x3b, 0xb5, 0xdd, 0xc7, 0xfe, 0x97,
+	0xe0, 0x19, 0x9d, 0xa0, 0x13, 0x18, 0x42, 0xe9, 0x20, 0xbe, 0x2f, 0x56, 0x62, 0x21, 0xfe, 0x1b,
-	0xff, 0xe1, 0xff, 0x6f, 0x48, 0xb6, 0x2d, 0xc4, 0xb6, 0x56, 0x2c, 0xc7, 0xdb, 0x36, 0xbd, 0xe1,
+	0x92, 0xad, 0x6b, 0x39, 0xd3, 0x56, 0x10, 0x71, 0xb3, 0x45, 0x83, 0xeb, 0xaf, 0x8d, 0x30, 0x1e,
-	0xf6, 0x6b, 0x23, 0x8c, 0x41, 0x94, 0xae, 0x66, 0x82, 0xf3, 0x44, 0xb2, 0x4f, 0x3b, 0xde, 0x02,
+	0xb1, 0xba, 0x8e, 0x0f, 0xd1, 0x03, 0x23, 0x3c, 0xec, 0xf0, 0x0d, 0x6a, 0x78, 0xc0, 0xc8, 0x24,
-	0x95, 0x0c, 0x60, 0x64, 0x92, 0x97, 0x94, 0xe5, 0xe5, 0xb8, 0x26, 0xd3, 0x0c, 0x18, 0x73, 0xb8,
+	0x2d, 0xa3, 0x4c, 0x2b, 0x8a, 0x7d, 0xe6, 0x78, 0x30, 0xa6, 0x70, 0xf5, 0xdf, 0x14, 0xa8, 0xd5,
-	0xf3, 0x4f, 0x0e, 0x54, 0xe9, 0x82, 0x41, 0xd2, 0x7e, 0xc2, 0x1f, 0x1c, 0x5a, 0xdb, 0xd3, 0x75,
+	0x05, 0x8f, 0x14, 0xf5, 0xb4, 0xef, 0x2a, 0x68, 0x75, 0xdb, 0x75, 0x03, 0x8b, 0x30, 0x27, 0xf0,
-	0x4b, 0x21, 0x4c, 0xb3, 0x4c, 0x0c, 0xd4, 0xd5, 0x19, 0x7f, 0x17, 0x95, 0x42, 0x6d, 0xde, 0x23,
+	0x31, 0xd0, 0xd8, 0x65, 0x6a, 0x80, 0x4e, 0xa7, 0xb9, 0x79, 0x9f, 0xf8, 0xb6, 0x0b, 0xb4, 0xa1,
-	0xa6, 0xaa, 0x43, 0x99, 0xab, 0x71, 0xf5, 0x65, 0xf9, 0x7f, 0x67, 0xfd, 0xea, 0xc2, 0xa0, 0x5f,
+	0xb4, 0xaa, 0x9b, 0xcb, 0x5b, 0x97, 0xf4, 0x69, 0xe9, 0xd3, 0xb1, 0xa4, 0x64, 0xbe, 0xf8, 0xb8,
-	0x2d, 0xe1, 0x84, 0x15, 0xa7, 0xd0, 0x7c, 0x1b, 0xad, 0x91, 0x1e, 0xd1, 0x74, 0xd2, 0xd6, 0xe1,
+	0xbf, 0xb1, 0x30, 0xe8, 0x6f, 0x9c, 0x96, 0xf7, 0x29, 0x2e, 0xb2, 0xab, 0x07, 0x68, 0x95, 0xf4,
-	0x81, 0x79, 0x64, 0xa9, 0x40, 0xcb, 0xb9, 0x1a, 0x57, 0x2f, 0x36, 0x6a, 0x62, 0x4c, 0x7f, 0x4f,
+	0x88, 0xe3, 0x92, 0x03, 0x17, 0x6e, 0xf9, 0x7b, 0x81, 0x0d, 0xb4, 0x51, 0x69, 0x29, 0x9b, 0xcb,
-	0x32, 0xb1, 0xb7, 0x23, 0x7a, 0x80, 0x16, 0xe8, 0xa0, 0x30, 0xcb, 0x91, 0x37, 0x07, 0xfd, 0xea,
+	0x5b, 0xad, 0xbc, 0x45, 0x9e, 0x63, 0xbd, 0x77, 0x45, 0xe7, 0x80, 0x36, 0xb8, 0x60, 0xb1, 0x20,
-	0xda, 0x5e, 0xca, 0x1b, 0x8f, 0xf1, 0xf1, 0x12, 0x5a, 0xa6, 0x5d, 0xe2, 0x80, 0xb7, 0x56, 0xce,
+	0x32, 0xcf, 0x0e, 0xfa, 0x1b, 0xab, 0xdb, 0x05, 0x6d, 0x3c, 0xc6, 0xa7, 0x1a, 0x68, 0x89, 0x76,
-	0xd7, 0xb8, 0xfa, 0x92, 0xbc, 0x1e, 0x84, 0xb7, 0xdc, 0x0a, 0x0d, 0x78, 0x84, 0x11, 0x7e, 0xc8,
+	0x49, 0x04, 0x7c, 0xaf, 0x51, 0x6d, 0x29, 0x9b, 0x8b, 0xe6, 0x5a, 0xe2, 0xe0, 0x52, 0x3b, 0x15,
-	0xa1, 0xcd, 0xa6, 0xa5, 0xb6, 0x94, 0x2e, 0xa8, 0xae, 0xae, 0x99, 0x9d, 0x7d, 0xcb, 0x64, 0xf0,
+	0xe0, 0x11, 0x46, 0xfb, 0xa9, 0x82, 0xce, 0xee, 0x07, 0x76, 0xdb, 0xea, 0x82, 0x1d, 0xbb, 0x8e,
-	0x15, 0xe3, 0x1f, 0xa1, 0x25, 0x2f, 0x0d, 0x2a, 0x61, 0xc4, 0x3f, 0x67, 0xb1, 0x71, 0x3b, 0x16,
+	0xdf, 0xb9, 0x1e, 0xf8, 0x0c, 0xbe, 0x66, 0xea, 0x3d, 0xb4, 0xc8, 0xeb, 0x66, 0x13, 0x46, 0x1a,
-	0x65, 0xa4, 0xa6, 0x68, 0x9f, 0x76, 0xbc, 0x05, 0x2a, 0x7a, 0x68, 0x2f, 0xee, 0x07, 0xed, 0xcf,
+	0x8a, 0xf0, 0xf2, 0x72, 0xce, 0xcb, 0x2c, 0xfd, 0x7a, 0x78, 0xd8, 0xe1, 0x1b, 0x54, 0xe7, 0x68,
-	0x41, 0x61, 0xf7, 0x81, 0x11, 0x99, 0x0f, 0xb6, 0x46, 0xa3, 0x35, 0x1c, 0xb1, 0xf2, 0x0f, 0x51,
+	0xee, 0xf7, 0xad, 0x83, 0x2f, 0xc0, 0x62, 0x37, 0x81, 0x11, 0x53, 0x4d, 0x4c, 0xa3, 0xd1, 0x1e,
-	0x81, 0xda, 0xa0, 0x04, 0x1a, 0xdc, 0x11, 0xa7, 0xdd, 0x41, 0x71, 0x52, 0x8c, 0x2d, 0x1b, 0x14,
+	0xce, 0x58, 0xd5, 0x3b, 0xa8, 0x46, 0x43, 0xb0, 0x92, 0x1c, 0x5c, 0x9d, 0x9e, 0xf5, 0x49, 0x3e,
-	0x79, 0x25, 0xd8, 0xa3, 0xe0, 0x7d, 0x61, 0x9f, 0x91, 0x7f, 0x84, 0xae, 0x51, 0x46, 0x98, 0x4b,
+	0xb6, 0x43, 0xb0, 0xcc, 0x53, 0x89, 0x8d, 0x1a, 0x5f, 0x61, 0xc1, 0xa8, 0xde, 0x43, 0x27, 0x29,
-	0x7d, 0x09, 0x8a, 0x8d, 0xdd, 0x2b, 0x70, 0xfb, 0xfe, 0x72, 0x29, 0x60, 0xbf, 0x36, 0xfc, 0xc6,
+	0x23, 0x2c, 0xa6, 0x22, 0x05, 0xcb, 0x5b, 0xd7, 0x8e, 0xc1, 0x2d, 0xf4, 0xcd, 0x7a, 0xc2, 0x7e,
-	0x01, 0xaf, 0xf0, 0x33, 0x87, 0xca, 0x93, 0xdc, 0x3e, 0xd0, 0x28, 0xe3, 0x3f, 0x1d, 0x93, 0x4e,
+	0x72, 0xb8, 0xc6, 0x09, 0xaf, 0xf6, 0xab, 0x82, 0x1a, 0x93, 0xd4, 0x3e, 0x74, 0x28, 0x53, 0x3f,
-	0xbc, 0x9c, 0x74, 0x9e, 0xb7, 0x2f, 0xdc, 0x5a, 0xb0, 0xed, 0x52, 0xb8, 0x12, 0x93, 0xed, 0x63,
+	0x1b, 0x4b, 0x9d, 0x7e, 0xb4, 0xd4, 0x71, 0x6d, 0x91, 0xb8, 0xd5, 0xc4, 0xec, 0x62, 0xba, 0x93,
-	0xb4, 0xa8, 0x31, 0x30, 0xbc, 0xbb, 0x93, 0xaf, 0x17, 0x1b, 0x8d, 0xd9, 0xcf, 0x26, 0xaf, 0x06,
+	0x4b, 0xdb, 0x27, 0xe8, 0x84, 0xc3, 0xc0, 0xe3, 0x67, 0x87, 0x9f, 0xd6, 0xad, 0xd9, 0x63, 0x33,
-	0xf4, 0x8b, 0x87, 0x1e, 0x11, 0x1e, 0xf2, 0x09, 0xcf, 0x32, 0xce, 0xe4, 0x09, 0xcb, 0xef, 0xa2,
+	0x57, 0x12, 0xfa, 0x13, 0xbb, 0x9c, 0x08, 0x0f, 0xf9, 0xb4, 0x47, 0x25, 0x31, 0xf1, 0xc4, 0xaa,
-	0x15, 0xea, 0x5f, 0x46, 0x50, 0xbd, 0x9b, 0x16, 0x5c, 0xfd, 0xcd, 0x80, 0x68, 0xa5, 0x15, 0xb3,
+	0xd7, 0xd0, 0x29, 0x2a, 0x0e, 0x23, 0xd8, 0xfc, 0xa4, 0x89, 0xb8, 0x96, 0xcc, 0xb3, 0x09, 0xd1,
-	0xe1, 0x04, 0x92, 0x7f, 0x03, 0x95, 0x6c, 0x8b, 0x81, 0xc9, 0x34, 0xa2, 0x87, 0x97, 0x3e, 0x5f,
+	0xa9, 0x76, 0x4e, 0x86, 0x25, 0xa4, 0xfa, 0x06, 0xaa, 0x87, 0x01, 0x03, 0x9f, 0x39, 0xc4, 0x4d,
-	0x5f, 0x96, 0x79, 0xaf, 0x64, 0x9a, 0x09, 0x0b, 0x4e, 0x21, 0x85, 0x6f, 0x39, 0xb4, 0x9d, 0x9d,
+	0x0f, 0x7d, 0x75, 0x73, 0xc9, 0x54, 0x07, 0xfd, 0x8d, 0xfa, 0xbe, 0x24, 0xc1, 0x05, 0xa4, 0xf6,
-	0x1d, 0xfe, 0xeb, 0x51, 0x45, 0xee, 0xeb, 0x44, 0x33, 0x68, 0x99, 0xf3, 0x35, 0x79, 0x73, 0xba,
+	0xbd, 0x82, 0xd6, 0xcb, 0xab, 0xa3, 0x7e, 0x83, 0xea, 0x69, 0xc4, 0xd7, 0x5d, 0xe2, 0x78, 0x69,
-	0x26, 0x38, 0xee, 0x33, 0xe2, 0x0e, 0x52, 0x3e, 0x56, 0xce, 0x43, 0x6a, 0x9c, 0xda, 0x4a, 0xf8,
+	0x07, 0xbf, 0x79, 0xb4, 0x0e, 0x16, 0x3a, 0x23, 0xee, 0xa4, 0xe4, 0x2f, 0x24, 0x31, 0xd5, 0x25,
-	0x2e, 0x87, 0x56, 0x13, 0x90, 0x39, 0x94, 0xcc, 0x87, 0x89, 0x92, 0x91, 0x66, 0x39, 0x66, 0x56,
+	0x18, 0xc5, 0x05, 0x53, 0xda, 0x0f, 0x15, 0xb4, 0x22, 0x41, 0xe6, 0xd0, 0x32, 0x1f, 0x49, 0x2d,
-	0xad, 0x9c, 0xa4, 0x6a, 0x65, 0x67, 0x16, 0xd2, 0xe9, 0x45, 0x32, 0xe0, 0x50, 0x25, 0x81, 0xdf,
+	0x63, 0xcc, 0x12, 0x66, 0x59, 0xaf, 0xdc, 0x2d, 0xf4, 0xca, 0x95, 0x59, 0x48, 0xa7, 0x37, 0xc9,
-	0xb7, 0x4c, 0xea, 0x1a, 0xe0, 0x60, 0x78, 0x0c, 0x0e, 0x98, 0x0a, 0xf0, 0x2f, 0xa3, 0x25, 0x62,
+	0x40, 0x41, 0x4d, 0x09, 0x7f, 0x3d, 0xf0, 0x69, 0xec, 0x41, 0x84, 0xe1, 0x3e, 0x44, 0xe0, 0x5b,
-	0x6b, 0xf7, 0x1c, 0xcb, 0xb5, 0x83, 0x2b, 0x15, 0x5d, 0xfd, 0xbd, 0xe6, 0xa1, 0xbf, 0x8e, 0x23,
+	0xa0, 0x5e, 0x42, 0x8b, 0x24, 0x74, 0x6e, 0x44, 0x41, 0x1c, 0x26, 0x47, 0x2a, 0x3b, 0xfa, 0xdb,
-	0x84, 0x87, 0x0e, 0x23, 0xf2, 0xa3, 0x8d, 0xa1, 0xc3, 0x7d, 0x70, 0x84, 0xe0, 0x6b, 0xa8, 0x60,
+	0xfb, 0xbb, 0x62, 0x1f, 0x67, 0x08, 0x8e, 0x4e, 0x3d, 0x12, 0xde, 0xe6, 0xd0, 0xa9, 0x1d, 0x9c,
-	0x12, 0x03, 0xca, 0x05, 0x1f, 0x19, 0x9d, 0xfd, 0x88, 0x18, 0x80, 0x7d, 0x0b, 0x2f, 0xa3, 0xbc,
+	0x21, 0xd4, 0x16, 0xaa, 0xf9, 0xc4, 0x83, 0x46, 0x4d, 0x20, 0xb3, 0xd8, 0xf7, 0x88, 0x07, 0x58,
-	0xab, 0xa9, 0xe5, 0x45, 0x1f, 0x70, 0x3b, 0x00, 0xe4, 0x8f, 0x0f, 0x0f, 0x5e, 0xf4, 0xab, 0xff,
+	0x48, 0x54, 0x13, 0x55, 0x63, 0xc7, 0x6e, 0x9c, 0x10, 0x80, 0xcb, 0x09, 0xa0, 0x7a, 0x7b, 0x77,
-	0xcf, 0x7a, 0x3a, 0xd8, 0x13, 0x1b, 0xa8, 0x78, 0x7c, 0x78, 0x80, 0x3d, 0x67, 0xe1, 0x47, 0x0e,
+	0xe7, 0x79, 0x7f, 0xe3, 0xa5, 0xb2, 0xbb, 0x86, 0x3d, 0x08, 0x81, 0xea, 0xb7, 0x77, 0x77, 0x30,
-	0xad, 0x27, 0x0e, 0x39, 0x87, 0x16, 0xd0, 0x4c, 0xb6, 0x80, 0x97, 0x66, 0x48, 0x59, 0x46, 0xed,
+	0x57, 0xd6, 0x7e, 0x56, 0xd0, 0x9a, 0x14, 0xe4, 0x1c, 0x46, 0xc0, 0xbe, 0x3c, 0x02, 0x5e, 0x9e,
-	0x7f, 0xc3, 0xa1, 0x5a, 0x02, 0xd7, 0x24, 0x0e, 0x31, 0x80, 0x81, 0x43, 0xaf, 0x9a, 0xac, 0x1a,
+	0xa1, 0x64, 0x25, 0xbd, 0xff, 0xad, 0x82, 0x5a, 0x12, 0x6e, 0x9f, 0x44, 0xc4, 0x03, 0x06, 0x11,
-	0x2a, 0x9c, 0x6a, 0xa6, 0xea, 0xdf, 0xd5, 0x98, 0xfc, 0xef, 0x6b, 0xa6, 0x8a, 0x7d, 0x4b, 0x94,
+	0x3d, 0x6e, 0xb1, 0x5a, 0xa8, 0x76, 0xe8, 0xf8, 0xb6, 0x38, 0xab, 0xb9, 0xf4, 0x7f, 0xe0, 0xf8,
-	0xa0, 0x7c, 0x56, 0x82, 0x84, 0xa7, 0x1c, 0xba, 0x35, 0xb5, 0x5a, 0x23, 0x0e, 0x2e, 0x33, 0xc9,
+	0x36, 0x16, 0x92, 0xac, 0x40, 0xd5, 0xb2, 0x02, 0x69, 0x0f, 0x15, 0x74, 0x71, 0x6a, 0xb7, 0x66,
-	0x6f, 0xa3, 0xeb, 0xae, 0x49, 0x5d, 0x8d, 0x79, 0xef, 0x5d, 0xbc, 0x01, 0x6d, 0x0c, 0xfa, 0xd5,
+	0x1c, 0x4a, 0x69, 0x91, 0xdf, 0x46, 0xa7, 0x63, 0x9f, 0xc6, 0x0e, 0xe3, 0xf7, 0x5d, 0x7e, 0x00,
-	0xeb, 0xc7, 0x49, 0x13, 0x4e, 0x63, 0x85, 0xef, 0x73, 0xa9, 0xfc, 0xfa, 0xed, 0xf0, 0x1e, 0x5a,
+	0x9d, 0xe1, 0xb7, 0xf6, 0x6d, 0x59, 0x84, 0x8b, 0x58, 0xed, 0xc7, 0x4a, 0xa1, 0xbe, 0x62, 0x1c,
-	0x8f, 0xb5, 0x03, 0x4a, 0x8f, 0x46, 0x31, 0xdc, 0x08, 0x62, 0x88, 0x7b, 0x0d, 0x01, 0x78, 0xdc,
+	0xde, 0x40, 0x6b, 0xb9, 0x71, 0x40, 0xe9, 0xde, 0xc8, 0x87, 0xf3, 0x89, 0x0f, 0x79, 0xad, 0x21,
-	0x87, 0xff, 0x12, 0xad, 0xda, 0x71, 0xa9, 0x83, 0xd2, 0xbe, 0x3b, 0x43, 0x4a, 0x27, 0xa4, 0x4a,
+	0x00, 0x8f, 0xeb, 0xa8, 0x5f, 0xa1, 0x95, 0x30, 0x9f, 0xea, 0xa4, 0xb5, 0xdf, 0x99, 0xa1, 0xa4,
-	0x5e, 0x1f, 0xf4, 0xab, 0xab, 0x09, 0x03, 0x4e, 0xee, 0xc3, 0x37, 0x51, 0x89, 0x44, 0x13, 0xce,
+	0x13, 0x4a, 0x65, 0xae, 0x0d, 0xfa, 0x1b, 0x2b, 0x92, 0x00, 0xcb, 0x76, 0xd4, 0x7d, 0x54, 0x27,
-	0x7d, 0xaf, 0xa5, 0x0f, 0xd3, 0x50, 0x0f, 0xdb, 0xdf, 0x5e, 0xc2, 0xfa, 0x62, 0x6c, 0x05, 0xa7,
+	0xd9, 0x93, 0xe8, 0x26, 0x1f, 0xe9, 0xc3, 0x32, 0x6c, 0xa6, 0xe3, 0x6f, 0x5b, 0x92, 0x3e, 0x1f,
-	0xfc, 0x85, 0x3f, 0x73, 0x68, 0x63, 0x42, 0x7b, 0xe0, 0x1b, 0x08, 0xa9, 0x8e, 0xd6, 0x03, 0x27,
+	0xdb, 0xc1, 0x05, 0x7d, 0xed, 0xaf, 0x0a, 0x3a, 0x33, 0x61, 0x3c, 0xa8, 0x5b, 0x08, 0xd9, 0x91,
-	0x26, 0x52, 0xd4, 0xe6, 0x0e, 0x22, 0x0b, 0x8e, 0xa1, 0xf8, 0xcf, 0x10, 0x1a, 0xb1, 0x07, 0x9a,
+	0xd3, 0x83, 0x28, 0x97, 0xa4, 0x6c, 0xcc, 0xed, 0x64, 0x12, 0x9c, 0x43, 0xa9, 0x9f, 0x23, 0x34,
-	0x88, 0xd3, 0x35, 0x49, 0xcf, 0x6b, 0x72, 0xc9, 0xe3, 0x8f, 0xad, 0xc6, 0x18, 0x79, 0x8a, 0x8a,
+	0x62, 0x4f, 0x72, 0xa2, 0x4f, 0xcf, 0x49, 0xf1, 0x81, 0x67, 0xd6, 0x39, 0x7f, 0x6e, 0x37, 0xc7,
-	0x0e, 0x50, 0x70, 0x7a, 0xa0, 0xbe, 0x6b, 0x39, 0xe5, 0xbc, 0x5f, 0x47, 0x6f, 0xcd, 0x20, 0xfa,
+	0xa8, 0x52, 0xb4, 0x1c, 0x01, 0x85, 0xa8, 0x07, 0xf6, 0x7b, 0x41, 0xd4, 0xa8, 0x8a, 0x3e, 0x7a,
-	0x58, 0x2b, 0x93, 0x37, 0x82, 0x23, 0x15, 0xf1, 0x88, 0x18, 0xc7, 0x77, 0xe1, 0x5b, 0x68, 0x4b,
+	0x6b, 0x86, 0xa4, 0x8f, 0x8d, 0x32, 0xf3, 0x4c, 0x12, 0xd2, 0x32, 0x1e, 0x11, 0xe3, 0xbc, 0x15,
-	0x05, 0x12, 0x0b, 0xf3, 0x0b, 0x17, 0x28, 0x03, 0xd5, 0xef, 0x50, 0x4b, 0xf2, 0xad, 0x80, 0x60,
+	0xb5, 0x8d, 0xce, 0xd9, 0x40, 0x72, 0x6e, 0x7e, 0x19, 0x03, 0x65, 0x60, 0x8b, 0x09, 0xb5, 0x68,
-	0xeb, 0x60, 0x12, 0x08, 0x4f, 0xf6, 0x15, 0x7e, 0xe5, 0xd0, 0x56, 0x22, 0xb2, 0x8f, 0xc0, 0xb0,
+	0x5e, 0x4c, 0x08, 0xce, 0xed, 0x4c, 0x02, 0xe1, 0xc9, 0xba, 0xda, 0xef, 0x0a, 0x3a, 0x27, 0x79,
-	0x75, 0xc2, 0x60, 0x0e, 0xcf, 0xd1, 0x49, 0xe2, 0x39, 0x7a, 0x7d, 0x06, 0xf9, 0xc2, 0x20, 0xb3,
+	0xf6, 0x31, 0x78, 0xa1, 0x4b, 0x18, 0xcc, 0xe1, 0x3a, 0xba, 0x2b, 0x5d, 0x47, 0xaf, 0xcf, 0x90,
-	0x9e, 0x25, 0xe1, 0x17, 0x0e, 0xdd, 0x98, 0xe8, 0x31, 0x87, 0xf6, 0xfa, 0x30, 0xd9, 0x5e, 0x5f,
+	0xbe, 0xd4, 0xc9, 0xb2, 0x6b, 0x49, 0xfb, 0x4d, 0x41, 0xe7, 0x27, 0x6a, 0xcc, 0x61, 0xbc, 0xde,
-	0xbd, 0xc2, 0xb9, 0x32, 0xda, 0xec, 0x79, 0xd6, 0xa9, 0x5a, 0xc3, 0xb1, 0xf5, 0xbf, 0x37, 0x3f,
+	0x91, 0xc7, 0xeb, 0xab, 0xc7, 0x88, 0xab, 0x64, 0xcc, 0x3e, 0x29, 0x8b, 0xaa, 0x3d, 0x7c, 0xb6,
-	0x08, 0x7f, 0x25, 0xc7, 0x20, 0x4a, 0xe7, 0x70, 0x8c, 0x64, 0x47, 0xc9, 0x5d, 0xaa, 0xa3, 0x8c,
+	0xfe, 0xff, 0xde, 0x0f, 0xda, 0xdf, 0xf2, 0x33, 0x88, 0xd2, 0x39, 0x84, 0x21, 0x4f, 0x94, 0xca,
-	0x35, 0xda, 0xfc, 0x8c, 0x8d, 0x96, 0xd2, 0xab, 0x35, 0xda, 0x13, 0xb4, 0x9a, 0x7c, 0x7d, 0x0a,
+	0x91, 0x26, 0xca, 0xd8, 0xa0, 0xad, 0xce, 0x38, 0x68, 0x29, 0x3d, 0xde, 0xa0, 0xbd, 0x8b, 0x56,
-	0x97, 0xfc, 0xcd, 0xe7, 0x53, 0xb7, 0x12, 0xaf, 0x53, 0x92, 0x29, 0x3d, 0x7b, 0x50, 0xfa, 0x6f,
+	0xe4, 0xdb, 0xa7, 0x76, 0xc4, 0x6f, 0x3e, 0x41, 0xdd, 0x96, 0x6e, 0x27, 0x99, 0xa9, 0xf8, 0xf6,
-	0x9e, 0x3d, 0x28, 0xcd, 0x28, 0x8a, 0x9f, 0x92, 0xb3, 0xc7, 0x44, 0x9d, 0xe7, 0x3f, 0x7b, 0x78,
+	0xa0, 0xf4, 0xbf, 0xfc, 0xf6, 0xa0, 0xb4, 0xa4, 0x29, 0x7e, 0x91, 0xdf, 0x1e, 0x13, 0xf3, 0x3c,
-	0x3f, 0xa5, 0xbd, 0x7f, 0xa9, 0x4d, 0x94, 0x70, 0x86, 0x8c, 0x7e, 0x4a, 0x1f, 0x85, 0x06, 0x3c,
+	0xff, 0xb7, 0x07, 0xff, 0x94, 0xe6, 0x7f, 0x69, 0x48, 0xac, 0xf4, 0x0d, 0x99, 0x7d, 0x4a, 0xef,
-	0xc2, 0xc8, 0xf2, 0xd9, 0x45, 0x65, 0xe1, 0xfc, 0xa2, 0xb2, 0xf0, 0xfc, 0xa2, 0xb2, 0xf0, 0x74,
+	0xa5, 0x02, 0x3c, 0xc2, 0x68, 0xf7, 0x51, 0x5d, 0xfe, 0x0d, 0xe0, 0x58, 0x37, 0x5f, 0x0b, 0xd5,
-	0x50, 0xe1, 0xce, 0x06, 0x15, 0xee, 0x7c, 0x50, 0xe1, 0x9e, 0x0f, 0x2a, 0xdc, 0x6f, 0x83, 0x0a,
+	0x44, 0xe5, 0x0a, 0xae, 0xef, 0x10, 0x46, 0xb0, 0x90, 0x98, 0xe6, 0xe3, 0x67, 0xcd, 0x85, 0x27,
-	0xf7, 0xec, 0xf7, 0xca, 0xc2, 0x27, 0x37, 0xa7, 0xfd, 0x61, 0xe6, 0xef, 0x00, 0x00, 0x00, 0xff,
+	0xcf, 0x9a, 0x0b, 0x4f, 0x9f, 0x35, 0x17, 0x1e, 0x0e, 0x9a, 0xca, 0xe3, 0x41, 0x53, 0x79, 0x32,
-	0xff, 0x94, 0x38, 0x0b, 0x13, 0xd0, 0x11, 0x00, 0x00,
+	0x68, 0x2a, 0x4f, 0x07, 0x4d, 0xe5, 0x8f, 0x41, 0x53, 0x79, 0xf4, 0x67, 0x73, 0xe1, 0xd3, 0x0b,
+	0xd3, 0x7e, 0x31, 0xfa, 0x27, 0x00, 0x00, 0xff, 0xff, 0x67, 0xe4, 0xf6, 0x18, 0x69, 0x12, 0x00,
+	0x00,
 }
 
 func (m *AllocationResult) Marshal() (dAtA []byte, err error) {
@@ -695,11 +726,20 @@ func (m *AllocationResult) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 		i--
 		dAtA[i] = 0x12
 	}
-	i -= len(m.ResourceHandle)
+	if len(m.ResourceHandles) > 0 {
-	copy(dAtA[i:], m.ResourceHandle)
+		for iNdEx := len(m.ResourceHandles) - 1; iNdEx >= 0; iNdEx-- {
-	i = encodeVarintGenerated(dAtA, i, uint64(len(m.ResourceHandle)))
+			{
-	i--
+				size, err := m.ResourceHandles[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-	dAtA[i] = 0xa
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintGenerated(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
 	return len(dAtA) - i, nil
 }
 
@@ -1487,6 +1527,39 @@ func (m *ResourceClassParametersReference) MarshalToSizedBuffer(dAtA []byte) (in
 	return len(dAtA) - i, nil
 }
 
+func (m *ResourceHandle) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ResourceHandle) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ResourceHandle) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	i -= len(m.Data)
+	copy(dAtA[i:], m.Data)
+	i = encodeVarintGenerated(dAtA, i, uint64(len(m.Data)))
+	i--
+	dAtA[i] = 0x12
+	i -= len(m.DriverName)
+	copy(dAtA[i:], m.DriverName)
+	i = encodeVarintGenerated(dAtA, i, uint64(len(m.DriverName)))
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
+}
+
 func encodeVarintGenerated(dAtA []byte, offset int, v uint64) int {
 	offset -= sovGenerated(v)
 	base := offset
@@ -1504,8 +1577,12 @@ func (m *AllocationResult) Size() (n int) {
 	}
 	var l int
 	_ = l
-	l = len(m.ResourceHandle)
+	if len(m.ResourceHandles) > 0 {
-	n += 1 + l + sovGenerated(uint64(l))
+		for _, e := range m.ResourceHandles {
+			l = e.Size()
+			n += 1 + l + sovGenerated(uint64(l))
+		}
+	}
 	if m.AvailableOnNodes != nil {
 		l = m.AvailableOnNodes.Size()
 		n += 1 + l + sovGenerated(uint64(l))
@@ -1796,6 +1873,19 @@ func (m *ResourceClassParametersReference) Size() (n int) {
 	return n
 }
 
+func (m *ResourceHandle) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.DriverName)
+	n += 1 + l + sovGenerated(uint64(l))
+	l = len(m.Data)
+	n += 1 + l + sovGenerated(uint64(l))
+	return n
+}
+
 func sovGenerated(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
 }
@@ -1806,8 +1896,13 @@ func (this *AllocationResult) String() string {
 	if this == nil {
 		return "nil"
 	}
+	repeatedStringForResourceHandles := "[]ResourceHandle{"
+	for _, f := range this.ResourceHandles {
+		repeatedStringForResourceHandles += strings.Replace(strings.Replace(f.String(), "ResourceHandle", "ResourceHandle", 1), `&`, ``, 1) + ","
+	}
+	repeatedStringForResourceHandles += "}"
 	s := strings.Join([]string{`&AllocationResult{`,
-		`ResourceHandle:` + fmt.Sprintf("%v", this.ResourceHandle) + `,`,
+		`ResourceHandles:` + repeatedStringForResourceHandles + `,`,
 		`AvailableOnNodes:` + strings.Replace(fmt.Sprintf("%v", this.AvailableOnNodes), "NodeSelector", "v1.NodeSelector", 1) + `,`,
 		`Shareable:` + fmt.Sprintf("%v", this.Shareable) + `,`,
 		`}`,
@@ -2042,6 +2137,17 @@ func (this *ResourceClassParametersReference) String() string {
 	}, "")
 	return s
 }
+func (this *ResourceHandle) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&ResourceHandle{`,
+		`DriverName:` + fmt.Sprintf("%v", this.DriverName) + `,`,
+		`Data:` + fmt.Sprintf("%v", this.Data) + `,`,
+		`}`,
+	}, "")
+	return s
+}
 func valueToStringGenerated(v interface{}) string {
 	rv := reflect.ValueOf(v)
 	if rv.IsNil() {
@@ -2081,9 +2187,9 @@ func (m *AllocationResult) Unmarshal(dAtA []byte) error {
 		switch fieldNum {
 		case 1:
 			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field ResourceHandle", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field ResourceHandles", wireType)
 			}
-			var stringLen uint64
+			var msglen int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
 					return ErrIntOverflowGenerated
@@ -2093,23 +2199,25 @@ func (m *AllocationResult) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
 			}
-			intStringLen := int(stringLen)
+			if msglen < 0 {
-			if intStringLen < 0 {
 				return ErrInvalidLengthGenerated
 			}
-			postIndex := iNdEx + intStringLen
+			postIndex := iNdEx + msglen
 			if postIndex < 0 {
 				return ErrInvalidLengthGenerated
 			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.ResourceHandle = string(dAtA[iNdEx:postIndex])
+			m.ResourceHandles = append(m.ResourceHandles, ResourceHandle{})
+			if err := m.ResourceHandles[len(m.ResourceHandles)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
 			iNdEx = postIndex
 		case 2:
 			if wireType != 2 {
@@ -4509,6 +4617,120 @@ func (m *ResourceClassParametersReference) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *ResourceHandle) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowGenerated
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ResourceHandle: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ResourceHandle: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field DriverName", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowGenerated
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.DriverName = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowGenerated
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Data = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipGenerated(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthGenerated
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipGenerated(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0

staging/src/k8s.io/api/resource/v1alpha2/generated.proto

@@ -29,21 +29,28 @@ import "k8s.io/apimachinery/pkg/runtime/schema/generated.proto";
 // Package-wide variables from generator "generated".
 option go_package = "k8s.io/api/resource/v1alpha2";
 
-// AllocationResult contains attributed of an allocated resource.
+// AllocationResult contains attributes of an allocated resource.
 message AllocationResult {
-  // ResourceHandle contains arbitrary data returned by the driver after a
+  // ResourceHandles contain the state associated with an allocation that
-  // successful allocation. This is opaque for
+  // should be maintained throughout the lifetime of a claim. Each
-  // Kubernetes. Driver documentation may explain to users how to
+  // ResourceHandle contains data that should be passed to a specific kubelet
-  // interpret this data if needed.
+  // plugin once it lands on a node. This data is returned by the driver
+  // after a successful allocation and is opaque to Kubernetes. Driver
+  // documentation may explain to users how to interpret this data if needed.
   //
-  // The maximum size of this field is 16KiB. This may get
+  // Setting this field is optional. It has a maximum size of 32 entries.
-  // increased in the future, but not reduced.
+  // If null (or empty), it is assumed this allocation will be processed by a
+  // single kubelet plugin with no ResourceHandle data attached. The name of
+  // the kubelet plugin invoked will match the DriverName set in the
+  // ResourceClaimStatus this AllocationResult is embedded in.
+  //
+  // +listType=atomic
   // +optional
-  optional string resourceHandle = 1;
+  repeated ResourceHandle resourceHandles = 1;
 
-  // This field will get set by the resource driver after it has
+  // This field will get set by the resource driver after it has allocated
-  // allocated the resource driver to inform the scheduler where it can
+  // the resource to inform the scheduler where it can schedule Pods using
-  // schedule Pods using the ResourceClaim.
+  // the ResourceClaim.
   //
   // Setting this field is optional. If null, the resource is available
   // everywhere.
@@ -235,9 +242,9 @@ message ResourceClaimStatus {
   // +optional
   optional string driverName = 1;
 
-  // Allocation is set by the resource driver once a resource has been
+  // Allocation is set by the resource driver once a resource or set of
-  // allocated successfully. If this is not specified, the resource is
+  // resources has been allocated successfully. If this is not specified, the
-  // not yet allocated.
+  // resources have not been allocated yet.
   // +optional
   optional AllocationResult allocation = 2;
 
@@ -370,3 +377,24 @@ message ResourceClassParametersReference {
   optional string namespace = 4;
 }
 
+// ResourceHandle holds opaque resource data for processing by a specific kubelet plugin.
+message ResourceHandle {
+  // DriverName specifies the name of the resource driver whose kubelet
+  // plugin should be invoked to process this ResourceHandle's data once it
+  // lands on a node. This may differ from the DriverName set in
+  // ResourceClaimStatus this ResourceHandle is embedded in.
+  optional string driverName = 1;
+
+  // Data contains the opaque data associated with this ResourceHandle. It is
+  // set by the controller component of the resource driver whose name
+  // matches the DriverName set in the ResourceClaimStatus this
+  // ResourceHandle is embedded in. It is set at allocation time and is
+  // intended for processing by the kubelet plugin whose name matches
+  // the DriverName set in this ResourceHandle.
+  //
+  // The maximum size of this field is 16KiB. This may get increased in the
+  // future, but not reduced.
+  // +optional
+  optional string data = 2;
+}
+

staging/src/k8s.io/api/resource/v1alpha2/types.go

@@ -99,9 +99,9 @@ type ResourceClaimStatus struct {
 	// +optional
 	DriverName string `json:"driverName,omitempty" protobuf:"bytes,1,opt,name=driverName"`
 
-	// Allocation is set by the resource driver once a resource has been
+	// Allocation is set by the resource driver once a resource or set of
-	// allocated successfully. If this is not specified, the resource is
+	// resources has been allocated successfully. If this is not specified, the
-	// not yet allocated.
+	// resources have not been allocated yet.
 	// +optional
 	Allocation *AllocationResult `json:"allocation,omitempty" protobuf:"bytes,2,opt,name=allocation"`
 
@@ -133,21 +133,28 @@ type ResourceClaimStatus struct {
 // claim.status.reservedFor.
 const ResourceClaimReservedForMaxSize = 32
 
-// AllocationResult contains attributed of an allocated resource.
+// AllocationResult contains attributes of an allocated resource.
 type AllocationResult struct {
-	// ResourceHandle contains arbitrary data returned by the driver after a
+	// ResourceHandles contain the state associated with an allocation that
-	// successful allocation. This is opaque for
+	// should be maintained throughout the lifetime of a claim. Each
-	// Kubernetes. Driver documentation may explain to users how to
+	// ResourceHandle contains data that should be passed to a specific kubelet
-	// interpret this data if needed.
+	// plugin once it lands on a node. This data is returned by the driver
+	// after a successful allocation and is opaque to Kubernetes. Driver
+	// documentation may explain to users how to interpret this data if needed.
 	//
-	// The maximum size of this field is 16KiB. This may get
+	// Setting this field is optional. It has a maximum size of 32 entries.
-	// increased in the future, but not reduced.
+	// If null (or empty), it is assumed this allocation will be processed by a
+	// single kubelet plugin with no ResourceHandle data attached. The name of
+	// the kubelet plugin invoked will match the DriverName set in the
+	// ResourceClaimStatus this AllocationResult is embedded in.
+	//
+	// +listType=atomic
 	// +optional
-	ResourceHandle string `json:"resourceHandle,omitempty" protobuf:"bytes,1,opt,name=resourceHandle"`
+	ResourceHandles []ResourceHandle `json:"resourceHandles,omitempty" protobuf:"bytes,1,opt,name=resourceHandles"`
 
-	// This field will get set by the resource driver after it has
+	// This field will get set by the resource driver after it has allocated
-	// allocated the resource driver to inform the scheduler where it can
+	// the resource to inform the scheduler where it can schedule Pods using
-	// schedule Pods using the ResourceClaim.
+	// the ResourceClaim.
 	//
 	// Setting this field is optional. If null, the resource is available
 	// everywhere.
@@ -160,8 +167,33 @@ type AllocationResult struct {
 	Shareable bool `json:"shareable,omitempty" protobuf:"varint,3,opt,name=shareable"`
 }
 
-// ResourceHandleMaxSize is the maximum size of allocation.resourceHandle.
+// AllocationResultResourceHandlesMaxSize represents the maximum number of
-const ResourceHandleMaxSize = 16 * 1024
+// entries in allocation.resourceHandles.
+const AllocationResultResourceHandlesMaxSize = 32
+
+// ResourceHandle holds opaque resource data for processing by a specific kubelet plugin.
+type ResourceHandle struct {
+	// DriverName specifies the name of the resource driver whose kubelet
+	// plugin should be invoked to process this ResourceHandle's data once it
+	// lands on a node. This may differ from the DriverName set in
+	// ResourceClaimStatus this ResourceHandle is embedded in.
+	DriverName string `json:"driverName,omitempty" protobuf:"bytes,1,opt,name=driverName"`
+
+	// Data contains the opaque data associated with this ResourceHandle. It is
+	// set by the controller component of the resource driver whose name
+	// matches the DriverName set in the ResourceClaimStatus this
+	// ResourceHandle is embedded in. It is set at allocation time and is
+	// intended for processing by the kubelet plugin whose name matches
+	// the DriverName set in this ResourceHandle.
+	//
+	// The maximum size of this field is 16KiB. This may get increased in the
+	// future, but not reduced.
+	// +optional
+	Data string `json:"data,omitempty" protobuf:"bytes,2,opt,name=data"`
+}
+
+// ResourceHandleDataMaxSize represents the maximum size of resourceHandle.data.
+const ResourceHandleDataMaxSize = 16 * 1024
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +k8s:prerelease-lifecycle-gen:introduced=1.26

staging/src/k8s.io/api/resource/v1alpha2/types_swagger_doc_generated.go

@@ -28,9 +28,9 @@ package v1alpha2
 
 // AUTO-GENERATED FUNCTIONS START HERE. DO NOT EDIT.
 var map_AllocationResult = map[string]string{
-	"":                 "AllocationResult contains attributed of an allocated resource.",
+	"":                 "AllocationResult contains attributes of an allocated resource.",
-	"resourceHandle":   "ResourceHandle contains arbitrary data returned by the driver after a successful allocation. This is opaque for Kubernetes. Driver documentation may explain to users how to interpret this data if needed.\n\nThe maximum size of this field is 16KiB. This may get increased in the future, but not reduced.",
+	"resourceHandles":  "ResourceHandles contain the state associated with an allocation that should be maintained throughout the lifetime of a claim. Each ResourceHandle contains data that should be passed to a specific kubelet plugin once it lands on a node. This data is returned by the driver after a successful allocation and is opaque to Kubernetes. Driver documentation may explain to users how to interpret this data if needed.\n\nSetting this field is optional. It has a maximum size of 32 entries. If null (or empty), it is assumed this allocation will be processed by a single kubelet plugin with no ResourceHandle data attached. The name of the kubelet plugin invoked will match the DriverName set in the ResourceClaimStatus this AllocationResult is embedded in.",
-	"availableOnNodes": "This field will get set by the resource driver after it has allocated the resource driver to inform the scheduler where it can schedule Pods using the ResourceClaim.\n\nSetting this field is optional. If null, the resource is available everywhere.",
+	"availableOnNodes": "This field will get set by the resource driver after it has allocated the resource to inform the scheduler where it can schedule Pods using the ResourceClaim.\n\nSetting this field is optional. If null, the resource is available everywhere.",
 	"shareable":        "Shareable determines whether the resource supports more than one consumer at a time.",
 }
 
@@ -146,7 +146,7 @@ func (ResourceClaimSpec) SwaggerDoc() map[string]string {
 var map_ResourceClaimStatus = map[string]string{
 	"":                      "ResourceClaimStatus tracks whether the resource has been allocated and what the resulting attributes are.",
 	"driverName":            "DriverName is a copy of the driver name from the ResourceClass at the time when allocation started.",
-	"allocation":            "Allocation is set by the resource driver once a resource has been allocated successfully. If this is not specified, the resource is not yet allocated.",
+	"allocation":            "Allocation is set by the resource driver once a resource or set of resources has been allocated successfully. If this is not specified, the resources have not been allocated yet.",
 	"reservedFor":           "ReservedFor indicates which entities are currently allowed to use the claim. A Pod which references a ResourceClaim which is not reserved for that Pod will not be started.\n\nThere can be at most 32 such reservations. This may get increased in the future, but not reduced.",
 	"deallocationRequested": "DeallocationRequested indicates that a ResourceClaim is to be deallocated.\n\nThe driver then must deallocate this claim and reset the field together with clearing the Allocation field.\n\nWhile DeallocationRequested is set, no new consumers may be added to ReservedFor.",
 }
@@ -219,4 +219,14 @@ func (ResourceClassParametersReference) SwaggerDoc() map[string]string {
 	return map_ResourceClassParametersReference
 }
 
+var map_ResourceHandle = map[string]string{
+	"":           "ResourceHandle holds opaque resource data for processing by a specific kubelet plugin.",
+	"driverName": "DriverName specifies the name of the resource driver whose kubelet plugin should be invoked to process this ResourceHandle's data once it lands on a node. This may differ from the DriverName set in ResourceClaimStatus this ResourceHandle is embedded in.",
+	"data":       "Data contains the opaque data associated with this ResourceHandle. It is set by the controller component of the resource driver whose name matches the DriverName set in the ResourceClaimStatus this ResourceHandle is embedded in. It is set at allocation time and is intended for processing by the kubelet plugin whose name matches the DriverName set in this ResourceHandle.\n\nThe maximum size of this field is 16KiB. This may get increased in the future, but not reduced.",
+}
+
+func (ResourceHandle) SwaggerDoc() map[string]string {
+	return map_ResourceHandle
+}
+
 // AUTO-GENERATED FUNCTIONS END HERE

staging/src/k8s.io/api/resource/v1alpha2/zz_generated.deepcopy.go

@@ -29,6 +29,11 @@ import (
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AllocationResult) DeepCopyInto(out *AllocationResult) {
 	*out = *in
+	if in.ResourceHandles != nil {
+		in, out := &in.ResourceHandles, &out.ResourceHandles
+		*out = make([]ResourceHandle, len(*in))
+		copy(*out, *in)
+	}
 	if in.AvailableOnNodes != nil {
 		in, out := &in.AvailableOnNodes, &out.AvailableOnNodes
 		*out = new(v1.NodeSelector)
@@ -475,3 +480,19 @@ func (in *ResourceClassParametersReference) DeepCopy() *ResourceClassParametersR
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ResourceHandle) DeepCopyInto(out *ResourceHandle) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceHandle.
+func (in *ResourceHandle) DeepCopy() *ResourceHandle {
+	if in == nil {
+		return nil
+	}
+	out := new(ResourceHandle)
+	in.DeepCopyInto(out)
+	return out
+}

staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha2.ResourceClaim.json

@@ -55,7 +55,12 @@
   "status": {
     "driverName": "driverNameValue",
     "allocation": {
-      "resourceHandle": "resourceHandleValue",
+      "resourceHandles": [
+        {
+          "driverName": "driverNameValue",
+          "data": "dataValue"
+        }
+      ],
       "availableOnNodes": {
         "nodeSelectorTerms": [
           {

staging/src/k8s.io/api/testdata/HEAD/resource.k8s.io.v1alpha2.ResourceClaim.yaml

@@ -53,7 +53,9 @@ status:
           operator: operatorValue
           values:
           - valuesValue
-    resourceHandle: resourceHandleValue
+    resourceHandles:
+    - data: dataValue
+      driverName: driverNameValue
     shareable: true
   deallocationRequested: true
   driverName: driverNameValue

staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go

@@ -11411,9 +11411,12 @@ var schemaYAML = typed.YAMLObject(`types:
     - name: availableOnNodes
       type:
         namedType: io.k8s.api.core.v1.NodeSelector
-    - name: resourceHandle
+    - name: resourceHandles
       type:
-        scalar: string
+        list:
+          elementType:
+            namedType: io.k8s.api.resource.v1alpha2.ResourceHandle
+          elementRelationship: atomic
     - name: shareable
       type:
         scalar: boolean
@@ -11627,6 +11630,15 @@ var schemaYAML = typed.YAMLObject(`types:
     - name: namespace
       type:
         scalar: string
+- name: io.k8s.api.resource.v1alpha2.ResourceHandle
+  map:
+    fields:
+    - name: data
+      type:
+        scalar: string
+    - name: driverName
+      type:
+        scalar: string
 - name: io.k8s.api.scheduling.v1.PriorityClass
   map:
     fields:

staging/src/k8s.io/client-go/applyconfigurations/resource/v1alpha2/allocationresult.go

@@ -25,7 +25,7 @@ import (
 // AllocationResultApplyConfiguration represents an declarative configuration of the AllocationResult type for use
 // with apply.
 type AllocationResultApplyConfiguration struct {
-	ResourceHandle   *string                            `json:"resourceHandle,omitempty"`
+	ResourceHandles  []ResourceHandleApplyConfiguration `json:"resourceHandles,omitempty"`
 	AvailableOnNodes *v1.NodeSelectorApplyConfiguration `json:"availableOnNodes,omitempty"`
 	Shareable        *bool                              `json:"shareable,omitempty"`
 }
@@ -36,11 +36,16 @@ func AllocationResult() *AllocationResultApplyConfiguration {
 	return &AllocationResultApplyConfiguration{}
 }
 
-// WithResourceHandle sets the ResourceHandle field in the declarative configuration to the given value
+// WithResourceHandles adds the given value to the ResourceHandles field in the declarative configuration
-// and returns the receiver, so that objects can be built by chaining "With" function invocations.
+// and returns the receiver, so that objects can be build by chaining "With" function invocations.
-// If called multiple times, the ResourceHandle field is set to the value of the last call.
+// If called multiple times, values provided by each call will be appended to the ResourceHandles field.
-func (b *AllocationResultApplyConfiguration) WithResourceHandle(value string) *AllocationResultApplyConfiguration {
+func (b *AllocationResultApplyConfiguration) WithResourceHandles(values ...*ResourceHandleApplyConfiguration) *AllocationResultApplyConfiguration {
-	b.ResourceHandle = &value
+	for i := range values {
+		if values[i] == nil {
+			panic("nil value passed to WithResourceHandles")
+		}
+		b.ResourceHandles = append(b.ResourceHandles, *values[i])
+	}
 	return b
 }
 

staging/src/k8s.io/client-go/applyconfigurations/resource/v1alpha2/resourcehandle.go

@@ -0,0 +1,48 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by applyconfiguration-gen. DO NOT EDIT.
+
+package v1alpha2
+
+// ResourceHandleApplyConfiguration represents an declarative configuration of the ResourceHandle type for use
+// with apply.
+type ResourceHandleApplyConfiguration struct {
+	DriverName *string `json:"driverName,omitempty"`
+	Data       *string `json:"data,omitempty"`
+}
+
+// ResourceHandleApplyConfiguration constructs an declarative configuration of the ResourceHandle type for use with
+// apply.
+func ResourceHandle() *ResourceHandleApplyConfiguration {
+	return &ResourceHandleApplyConfiguration{}
+}
+
+// WithDriverName sets the DriverName field in the declarative configuration to the given value
+// and returns the receiver, so that objects can be built by chaining "With" function invocations.
+// If called multiple times, the DriverName field is set to the value of the last call.
+func (b *ResourceHandleApplyConfiguration) WithDriverName(value string) *ResourceHandleApplyConfiguration {
+	b.DriverName = &value
+	return b
+}
+
+// WithData sets the Data field in the declarative configuration to the given value
+// and returns the receiver, so that objects can be built by chaining "With" function invocations.
+// If called multiple times, the Data field is set to the value of the last call.
+func (b *ResourceHandleApplyConfiguration) WithData(value string) *ResourceHandleApplyConfiguration {
+	b.Data = &value
+	return b
+}

staging/src/k8s.io/client-go/applyconfigurations/utils.go

@@ -1483,6 +1483,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} {
 		return &resourcev1alpha2.ResourceClassApplyConfiguration{}
 	case v1alpha2.SchemeGroupVersion.WithKind("ResourceClassParametersReference"):
 		return &resourcev1alpha2.ResourceClassParametersReferenceApplyConfiguration{}
+	case v1alpha2.SchemeGroupVersion.WithKind("ResourceHandle"):
+		return &resourcev1alpha2.ResourceHandleApplyConfiguration{}
 
 		// Group=scheduling.k8s.io, Version=v1
 	case schedulingv1.SchemeGroupVersion.WithKind("PriorityClass"):

test/e2e/dra/test-driver/app/controller.go

@@ -159,7 +159,7 @@ func (c *ExampleController) Allocate(ctx context.Context, claim *resourcev1alpha
 	return c.allocate(ctx, claim, claimParameters, class, classParameters, selectedNode)
 }
 
-// allocate simply copies parameters as JSON map into ResourceHandle.
+// allocate simply copies parameters as JSON map into a ResourceHandle.
 func (c *ExampleController) allocate(ctx context.Context, claim *resourcev1alpha2.ResourceClaim, claimParameters interface{}, class *resourcev1alpha2.ResourceClass, classParameters interface{}, selectedNode string) (result *resourcev1alpha2.AllocationResult, err error) {
 	logger := klog.LoggerWithValues(klog.LoggerWithName(klog.FromContext(ctx), "Allocate"), "claim", klog.KObj(claim), "uid", claim.UID)
 	defer func() {
@@ -223,7 +223,12 @@ func (c *ExampleController) allocate(ctx context.Context, claim *resourcev1alpha
 	if err != nil {
 		return nil, fmt.Errorf("encode parameters: %w", err)
 	}
-	allocation.ResourceHandle = string(data)
+	allocation.ResourceHandles = []resourcev1alpha2.ResourceHandle{
+		{
+			DriverName: c.driverName,
+			Data:       string(data),
+		},
+	}
 	var nodes []string
 	if node != "" {
 		nodes = append(nodes, node)