From ece1d76e8023683f76d42c452fe85b89663e6af5 Mon Sep 17 00:00:00 2001 From: Cici Huang Date: Wed, 12 Mar 2025 14:37:22 -0700 Subject: [PATCH] API manual changes Co-authored-by: Morten Torkildsen --- pkg/apis/resource/types.go | 148 +++++++++- pkg/apis/resource/validation/validation.go | 145 +++++++++- .../validation_resourceslice_test.go | 261 +++++++++++++++++- .../resource/resourceslice/strategy.go | 43 +++ .../resource/resourceslice/strategy_test.go | 194 ++++++++++++- .../src/k8s.io/api/resource/v1alpha3/types.go | 133 ++++++++- .../src/k8s.io/api/resource/v1beta1/types.go | 127 ++++++++- .../dynamic-resource-allocation/api/types.go | 36 ++- 8 files changed, 1054 insertions(+), 33 deletions(-) diff --git a/pkg/apis/resource/types.go b/pkg/apis/resource/types.go index f53402fa8a5..dcc9503f49c 100644 --- a/pkg/apis/resource/types.go +++ b/pkg/apis/resource/types.go @@ -106,7 +106,7 @@ type ResourceSliceSpec struct { // new nodes of the same type as some old node might also make new // resources available. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // This field is immutable. // // +optional @@ -118,7 +118,7 @@ type ResourceSliceSpec struct { // // Must use exactly one term. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // // +optional // +oneOf=NodeSelection @@ -126,7 +126,7 @@ type ResourceSliceSpec struct { // AllNodes indicates that all nodes have access to the resources in the pool. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // // +optional // +oneOf=NodeSelection @@ -139,6 +139,54 @@ type ResourceSliceSpec struct { // +optional // +listType=atomic Devices []Device + + // PerDeviceNodeSelection defines whether the access from nodes to + // resources in the pool is set on the ResourceSlice level or on each + // device. If it is set to true, every device defined the ResourceSlice + // must specify this individually. + // + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. + // + // +optional + // +oneOf=NodeSelection + // +featureGate=DRAPartitionableDevices + PerDeviceNodeSelection *bool + + // SharedCounters defines a list of counter sets, each of which + // has a name and a list of counters available. + // + // The names of the SharedCounters must be unique in the ResourceSlice. + // + // The maximum number of SharedCounters is 32. + // + // +optional + // +listType=atomic + // +featureGate=DRAPartitionableDevices + SharedCounters []CounterSet +} + +// CounterSet defines a named set of counters +// that are available to be used by devices defined in the +// ResourceSlice. +// +// The counters are not allocatable by themselves, but +// can be referenced by devices. When a device is allocated, +// the portion of counters it uses will no longer be available for use +// by other devices. +type CounterSet struct { + // Name defines the name of the counter set. + // It must be a DNS label. + // + // +required + Name string + + // Counters defines the set of counters for this CounterSet + // The name of each counter must be unique in that set and must be a DNS label. + // + // The maximum number of counters is 32. + // + // +required + Counters map[string]Counter } // DriverNameMaxLength is the maximum valid length of a driver name in the @@ -186,6 +234,28 @@ const ResourceSliceMaxSharedCapacity = 128 const ResourceSliceMaxDevices = 128 const PoolNameMaxLength = validation.DNS1123SubdomainMaxLength // Same as for a single node name. +// Defines the max number of SharedCounters that can be specified +// in a ResourceSlice. This is used to validate the fields: +// * spec.sharedCounters +const ResourceSliceMaxSharedCounters = 32 + +// Defines the max number of Counters from which a device +// can consume. This is used to validate the fields: +// * spec.devices[].consumesCounter +const ResourceSliceMaxDeviceCounterConsumptions = 32 + +// Defines the max number of counters +// that can be specified for sharedCounters in a ResourceSlice. +// This is used to validate the fields: +// * spec.sharedCounters[].counters +const ResourceSliceMaxSharedCountersCounters = 32 + +// Defines the max number of counters +// that can be specified for consumesCounter in a ResourceSlice. +// This is used to validate the fields: +// * spec.devices[].consumesCounter[].counters +const ResourceSliceMaxDeviceCounterConsumptionCounters = 32 + // Device represents one individual hardware instance that can be selected based // on its attributes. Besides the name, exactly one field must be set. type Device struct { @@ -220,6 +290,51 @@ type BasicDevice struct { // +optional Capacity map[QualifiedName]DeviceCapacity + // ConsumesCounter defines a list of references to sharedCounters + // and the set of counters that the device will + // consume from those counter sets. + // + // There can only be a single entry per counterSet. + // + // The maximum number of device counter consumption entries + // is 32. This is the same as the maximum number of shared counters + // allowed in a ResourceSlice. + // + // +optional + // +listType=atomic + // +featureGate=DRAPartitionableDevices + ConsumesCounter []DeviceCounterConsumption + + // NodeName identifies the node where the device is available. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + // +featureGate=DRAPartitionableDevices + NodeName *string + + // NodeSelector defines the nodes where the device is available. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + // +featureGate=DRAPartitionableDevices + NodeSelector *core.NodeSelector + + // AllNodes indicates that all nodes have access to the device. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + // +featureGate=DRAPartitionableDevices + AllNodes *bool + // If specified, these are the driver-defined taints. // // The maximum number of taints is 8. @@ -233,6 +348,25 @@ type BasicDevice struct { Taints []DeviceTaint } +// DeviceCounterConsumption defines a set of counters that +// a device will consume from a CounterSet. +type DeviceCounterConsumption struct { + // SharedCounter defines the shared counter from which the + // counters defined will be consumed. + // + // +required + SharedCounter string + + // Counters defines the Counter that will be consumed by + // the device. + // + // + // The maximum number of Counters is 32. + // + // +required + Counters map[string]Counter +} + // DeviceCapacity describes a quantity associated with a device. type DeviceCapacity struct { // Value defines how much of a certain device capacity is available. @@ -244,6 +378,14 @@ type DeviceCapacity struct { // capacity (= share a single device between different consumers). } +// Counter describes a quantity associated with a device. +type Counter struct { + // Value defines how much of a certain device counter is available. + // + // +required + Value resource.Quantity +} + // Limit for the sum of the number of entries in both attributes and capacity. const ResourceSliceMaxAttributesAndCapacitiesPerDevice = 32 diff --git a/pkg/apis/resource/validation/validation.go b/pkg/apis/resource/validation/validation.go index 2148455964c..4d10005a523 100644 --- a/pkg/apis/resource/validation/validation.go +++ b/pkg/apis/resource/validation/validation.go @@ -46,6 +46,10 @@ var ( validateDeviceName = corevalidation.ValidateDNS1123Label validateDeviceClassName = corevalidation.ValidateDNS1123Subdomain validateRequestName = corevalidation.ValidateDNS1123Label + validateCounterName = corevalidation.ValidateDNS1123Label + + // this is the max length limit for domain/ID + attributeAndCapacityMaxKeyLength = resource.DeviceMaxDomainLength + 1 + resource.DeviceMaxIDLength ) func validatePoolName(name string, fldPath *field.Path) field.ErrorList { @@ -582,7 +586,7 @@ func validateResourceSliceSpec(spec, oldSpec *resource.ResourceSliceSpec, fldPat allErrs = append(allErrs, apimachineryvalidation.ValidateImmutableField(spec.NodeName, oldSpec.NodeName, fldPath.Child("nodeName"))...) } - setFields := make([]string, 0, 3) + setFields := make([]string, 0, 4) if spec.NodeName != "" { setFields = append(setFields, "`nodeName`") allErrs = append(allErrs, validateNodeName(spec.NodeName, fldPath.Child("nodeName"))...) @@ -599,23 +603,71 @@ func validateResourceSliceSpec(spec, oldSpec *resource.ResourceSliceSpec, fldPat if spec.AllNodes { setFields = append(setFields, "`allNodes`") } + if spec.PerDeviceNodeSelection != nil { + if *spec.PerDeviceNodeSelection { + setFields = append(setFields, "`perDeviceNodeSelection`") + } else { + allErrs = append(allErrs, field.Invalid(fldPath.Child("perDeviceNodeSelection"), *spec.PerDeviceNodeSelection, + "must be either unset or set to true")) + } + + } switch len(setFields) { case 0: - allErrs = append(allErrs, field.Required(fldPath, "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required")) + allErrs = append(allErrs, field.Required(fldPath, "exactly one of `nodeName`, `nodeSelector`, `allNodes`, `perDeviceNodeSelection` is required")) case 1: default: allErrs = append(allErrs, field.Invalid(fldPath, fmt.Sprintf("{%s}", strings.Join(setFields, ", ")), - "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required, but multiple fields are set")) + "exactly one of `nodeName`, `nodeSelector`, `allNodes`, `perDeviceNodeSelection` is required, but multiple fields are set")) } - allErrs = append(allErrs, validateSet(spec.Devices, resource.ResourceSliceMaxDevices, validateDevice, + sharedCounterToCounterNames := gatherSharedCounterCounterNames(spec.SharedCounters) + allErrs = append(allErrs, validateSet(spec.Devices, resource.ResourceSliceMaxDevices, + func(device resource.Device, fldPath *field.Path) field.ErrorList { + return validateDevice(device, fldPath, sharedCounterToCounterNames, spec.PerDeviceNodeSelection) + }, func(device resource.Device) (string, string) { return device.Name, "name" }, fldPath.Child("devices"))...) + allErrs = append(allErrs, validateSet(spec.SharedCounters, resource.ResourceSliceMaxSharedCounters, + validateCounterSet, + func(counterSet resource.CounterSet) (string, string) { + return counterSet.Name, "name" + }, fldPath.Child("sharedCounters"))...) + return allErrs } +func validateCounterSet(counterSet resource.CounterSet, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if counterSet.Name == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("name"), "")) + } else { + allErrs = append(allErrs, validateCounterName(counterSet.Name, fldPath.Child("name"))...) + } + if len(counterSet.Counters) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("counters"), "")) + } else { + allErrs = append(allErrs, validateMap(counterSet.Counters, resource.ResourceSliceMaxSharedCountersCounters, attributeAndCapacityMaxKeyLength, + validateCounterName, validateDeviceCounter, fldPath.Child("counters"))...) + } + + return allErrs +} + +func gatherSharedCounterCounterNames(sharedCounters []resource.CounterSet) map[string]sets.Set[string] { + sharedCounterToCounterMap := make(map[string]sets.Set[string]) + for _, sharedCounter := range sharedCounters { + counterNames := sets.New[string]() + for counterName := range sharedCounter.Counters { + counterNames.Insert(counterName) + } + sharedCounterToCounterMap[sharedCounter.Name] = counterNames + } + return sharedCounterToCounterMap +} + func validateResourcePool(pool resource.ResourcePool, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, validatePoolName(pool.Name, fldPath.Child("name"))...) @@ -628,30 +680,98 @@ func validateResourcePool(pool resource.ResourcePool, fldPath *field.Path) field return allErrs } -func validateDevice(device resource.Device, fldPath *field.Path) field.ErrorList { +func validateDevice(device resource.Device, fldPath *field.Path, sharedCounterToCounterNames map[string]sets.Set[string], perDeviceNodeSelection *bool) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, validateDeviceName(device.Name, fldPath.Child("name"))...) if device.Basic == nil { allErrs = append(allErrs, field.Required(fldPath.Child("basic"), "")) } else { - allErrs = append(allErrs, validateBasicDevice(*device.Basic, fldPath.Child("basic"))...) + allErrs = append(allErrs, validateBasicDevice(*device.Basic, fldPath.Child("basic"), sharedCounterToCounterNames, perDeviceNodeSelection)...) } return allErrs } -func validateBasicDevice(device resource.BasicDevice, fldPath *field.Path) field.ErrorList { +func validateBasicDevice(device resource.BasicDevice, fldPath *field.Path, sharedCounterToCounterNames map[string]sets.Set[string], perDeviceNodeSelection *bool) field.ErrorList { var allErrs field.ErrorList // Warn about exceeding the maximum length only once. If any individual // field is too large, then so is the combination. - maxKeyLen := resource.DeviceMaxDomainLength + 1 + resource.DeviceMaxIDLength - allErrs = append(allErrs, validateMap(device.Attributes, -1, maxKeyLen, validateQualifiedName, validateDeviceAttribute, fldPath.Child("attributes"))...) - allErrs = append(allErrs, validateMap(device.Capacity, -1, maxKeyLen, validateQualifiedName, validateDeviceCapacity, fldPath.Child("capacity"))...) + allErrs = append(allErrs, validateMap(device.Attributes, -1, attributeAndCapacityMaxKeyLength, validateQualifiedName, validateDeviceAttribute, fldPath.Child("attributes"))...) + allErrs = append(allErrs, validateMap(device.Capacity, -1, attributeAndCapacityMaxKeyLength, validateQualifiedName, validateDeviceCapacity, fldPath.Child("capacity"))...) if combinedLen, max := len(device.Attributes)+len(device.Capacity), resource.ResourceSliceMaxAttributesAndCapacitiesPerDevice; combinedLen > max { allErrs = append(allErrs, field.Invalid(fldPath, combinedLen, fmt.Sprintf("the total number of attributes and capacities must not exceed %d", max))) } for i, taint := range device.Taints { allErrs = append(allErrs, validateDeviceTaint(taint, fldPath.Child("taints").Index(i))...) } + + allErrs = append(allErrs, validateSet(device.ConsumesCounter, resource.ResourceSliceMaxDeviceCounterConsumptions, + validateDeviceCounterConsumption, + func(deviceCapacityConsumption resource.DeviceCounterConsumption) (string, string) { + return deviceCapacityConsumption.SharedCounter, "sharedCounter" + }, fldPath.Child("consumesCounter"))...) + + for i, deviceCounterConsumption := range device.ConsumesCounter { + if capacityNames, exists := sharedCounterToCounterNames[deviceCounterConsumption.SharedCounter]; exists { + for capacityName := range deviceCounterConsumption.Counters { + if !capacityNames.Has(string(capacityName)) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("consumesCounter").Index(i).Child("counters"), + capacityName, "must reference a counter defined in the ResourceSlice sharedCounters")) + } + } + } else { + allErrs = append(allErrs, field.Invalid(fldPath.Child("consumesCounter").Index(i).Child("sharedCounter"), + deviceCounterConsumption.SharedCounter, "must reference a counterSet defined in the ResourceSlice sharedCounters")) + } + } + + if perDeviceNodeSelection != nil && *perDeviceNodeSelection { + setFields := make([]string, 0, 3) + if device.NodeName != nil { + if len(*device.NodeName) != 0 { + setFields = append(setFields, "`nodeName`") + allErrs = append(allErrs, validateNodeName(*device.NodeName, fldPath.Child("nodeName"))...) + } else { + allErrs = append(allErrs, field.Invalid(fldPath.Child("nodeName"), *device.NodeName, "must not be empty")) + } + + } + if device.NodeSelector != nil { + setFields = append(setFields, "`nodeSelector`") + allErrs = append(allErrs, corevalidation.ValidateNodeSelector(device.NodeSelector, false, fldPath.Child("nodeSelector"))...) + } + if device.AllNodes != nil { + if *device.AllNodes { + setFields = append(setFields, "`allNodes`") + } else { + allErrs = append(allErrs, field.Invalid(fldPath.Child("allNodes"), *device.AllNodes, "must be either unset or set to true")) + } + } + switch len(setFields) { + case 0: + allErrs = append(allErrs, field.Required(fldPath, "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required when `perDeviceNodeSelection` is set to true in the ResourceSlice spec")) + case 1: + default: + allErrs = append(allErrs, field.Invalid(fldPath, fmt.Sprintf("{%s}", strings.Join(setFields, ", ")), "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required when `perDeviceNodeSelection` is set to true in the ResourceSlice spec")) + } + } else if (perDeviceNodeSelection == nil || !*perDeviceNodeSelection) && (device.NodeName != nil || device.NodeSelector != nil || device.AllNodes != nil) { + allErrs = append(allErrs, field.Invalid(fldPath, nil, "`nodeName`, `nodeSelector` and `allNodes` can only be set if `perDeviceNodeSelection` is set to true in the ResourceSlice spec")) + } + + return allErrs +} + +func validateDeviceCounterConsumption(deviceCounterConsumption resource.DeviceCounterConsumption, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + if len(deviceCounterConsumption.SharedCounter) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("sharedCounter"), "")) + } + if deviceCounterConsumption.Counters == nil { + allErrs = append(allErrs, field.Required(fldPath.Child("counters"), "")) + } else { + allErrs = append(allErrs, validateMap(deviceCounterConsumption.Counters, resource.ResourceSliceMaxDeviceCounterConsumptionCounters, attributeAndCapacityMaxKeyLength, + validateCounterName, validateDeviceCounter, fldPath.Child("counters"))...) + } return allErrs } @@ -728,6 +848,11 @@ func validateDeviceCapacity(capacity resource.DeviceCapacity, fldPath *field.Pat return nil } +func validateDeviceCounter(counter resource.Counter, fldPath *field.Path) field.ErrorList { + // Any parsed quantity is valid. + return nil +} + func validateQualifiedName(name resource.QualifiedName, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if name == "" { diff --git a/pkg/apis/resource/validation/validation_resourceslice_test.go b/pkg/apis/resource/validation/validation_resourceslice_test.go index 34be623bc6c..6a487874f88 100644 --- a/pkg/apis/resource/validation/validation_resourceslice_test.go +++ b/pkg/apis/resource/validation/validation_resourceslice_test.go @@ -44,6 +44,12 @@ func testCapacity() map[resourceapi.QualifiedName]resourceapi.DeviceCapacity { } } +func testCounters() map[string]resourceapi.Counter { + return map[string]resourceapi.Counter{ + "memory": {Value: resource.MustParse("1Gi")}, + } +} + func testResourceSlice(name, nodeName, driverName string, numDevices int) *resourceapi.ResourceSlice { slice := &resourceapi.ResourceSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -273,7 +279,7 @@ func TestValidateResourceSlice(t *testing.T) { }(), }, "bad-node-selection": { - wantFailures: field.ErrorList{field.Invalid(field.NewPath("spec"), "{`nodeName`, `nodeSelector`}", "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required, but multiple fields are set")}, + wantFailures: field.ErrorList{field.Invalid(field.NewPath("spec"), "{`nodeName`, `nodeSelector`}", "exactly one of `nodeName`, `nodeSelector`, `allNodes`, `perDeviceNodeSelection` is required, but multiple fields are set")}, slice: func() *resourceapi.ResourceSlice { slice := testResourceSlice(goodName, goodName, driverName, 1) slice.Spec.NodeName = "worker" @@ -284,7 +290,7 @@ func TestValidateResourceSlice(t *testing.T) { }(), }, "bad-node-selection-all-nodes": { - wantFailures: field.ErrorList{field.Invalid(field.NewPath("spec"), "{`nodeName`, `allNodes`}", "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required, but multiple fields are set")}, + wantFailures: field.ErrorList{field.Invalid(field.NewPath("spec"), "{`nodeName`, `allNodes`}", "exactly one of `nodeName`, `nodeSelector`, `allNodes`, `perDeviceNodeSelection` is required, but multiple fields are set")}, slice: func() *resourceapi.ResourceSlice { slice := testResourceSlice(goodName, goodName, driverName, 1) slice.Spec.NodeName = "worker" @@ -293,7 +299,7 @@ func TestValidateResourceSlice(t *testing.T) { }(), }, "empty-node-selection": { - wantFailures: field.ErrorList{field.Required(field.NewPath("spec"), "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required")}, + wantFailures: field.ErrorList{field.Required(field.NewPath("spec"), "exactly one of `nodeName`, `nodeSelector`, `allNodes`, `perDeviceNodeSelection` is required")}, slice: func() *resourceapi.ResourceSlice { slice := testResourceSlice(goodName, goodName, driverName, 1) slice.Spec.NodeName = "" @@ -488,6 +494,233 @@ func TestValidateResourceSlice(t *testing.T) { return slice }(), }, + "bad-PerDeviceNodeSelection": { + wantFailures: field.ErrorList{ + field.Invalid(field.NewPath("spec"), "{`nodeName`, `perDeviceNodeSelection`}", "exactly one of `nodeName`, `nodeSelector`, `allNodes`, `perDeviceNodeSelection` is required, but multiple fields are set"), + field.Required(field.NewPath("spec", "devices").Index(0).Child("basic"), "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required when `perDeviceNodeSelection` is set to true in the ResourceSlice spec"), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.NodeName = "worker" + slice.Spec.PerDeviceNodeSelection = func() *bool { + r := true + return &r + }() + return slice + }(), + }, + "invalid-false-PerDeviceNodeSelection": { + wantFailures: field.ErrorList{ + field.Invalid(field.NewPath("spec", "perDeviceNodeSelection"), false, "must be either unset or set to true"), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.NodeName = "worker" + slice.Spec.PerDeviceNodeSelection = func() *bool { + r := false + return &r + }() + return slice + }(), + }, + "invalid-node-selector-in-basicdevice": { + wantFailures: field.ErrorList{ + field.Invalid(field.NewPath("spec", "devices").Index(0).Child("basic", "nodeName"), "", "must not be empty"), + field.Invalid(field.NewPath("spec", "devices").Index(0).Child("basic", "allNodes"), false, "must be either unset or set to true"), + field.Required(field.NewPath("spec", "devices").Index(0).Child("basic"), "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required when `perDeviceNodeSelection` is set to true in the ResourceSlice spec"), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.PerDeviceNodeSelection = func() *bool { + r := true + return &r + }() + slice.Spec.NodeName = "" + slice.Spec.Devices[0].Basic.NodeName = func() *string { + r := "" + return &r + }() + slice.Spec.Devices[0].Basic.AllNodes = func() *bool { + r := false + return &r + }() + return slice + }(), + }, + "bad-node-selector-in-basicdevice": { + wantFailures: field.ErrorList{ + field.Invalid(field.NewPath("spec", "devices").Index(0).Child("basic"), "{`nodeName`, `allNodes`}", "exactly one of `nodeName`, `nodeSelector`, or `allNodes` is required when `perDeviceNodeSelection` is set to true in the ResourceSlice spec"), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.PerDeviceNodeSelection = func() *bool { + r := true + return &r + }() + slice.Spec.NodeName = "" + slice.Spec.Devices[0].Basic.NodeName = func() *string { + r := "worker" + return &r + }() + slice.Spec.Devices[0].Basic.AllNodes = func() *bool { + r := true + return &r + }() + return slice + }(), + }, + "bad-name-shared-counters": { + wantFailures: field.ErrorList{ + field.Invalid(field.NewPath("spec", "sharedCounters").Index(0).Child("name"), badName, "a lowercase RFC 1123 label must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?')"), + field.Required(field.NewPath("spec", "sharedCounters").Index(0).Child("counters"), ""), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = []resourceapi.CounterSet{ + { + Name: badName, + }, + } + return slice + }(), + }, + "bad-countername-shared-counters": { + wantFailures: field.ErrorList{ + field.Invalid(field.NewPath("spec", "sharedCounters").Index(0).Child("counters").Key(badName), badName, "a lowercase RFC 1123 label must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?')"), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = []resourceapi.CounterSet{ + { + Name: goodName, + Counters: map[string]resourceapi.Counter{ + badName: {Value: resource.MustParse("1Gi")}, + }, + }, + } + return slice + }(), + }, + "missing-name-shared-counters": { + wantFailures: field.ErrorList{ + field.Required(field.NewPath("spec", "sharedCounters").Index(0).Child("name"), ""), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = []resourceapi.CounterSet{ + { + Counters: testCounters(), + }, + } + return slice + }(), + }, + "duplicate-shared-counters": { + wantFailures: field.ErrorList{ + field.Duplicate(field.NewPath("spec", "sharedCounters").Index(1).Child("name"), goodName), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = []resourceapi.CounterSet{ + { + Name: goodName, + Counters: testCounters(), + }, + { + Name: goodName, + Counters: testCounters(), + }, + } + return slice + }(), + }, + "too-large-shared-counters": { + wantFailures: field.ErrorList{ + field.TooMany(field.NewPath("spec", "sharedCounters"), resourceapi.ResourceSliceMaxSharedCounters+1, resourceapi.ResourceSliceMaxSharedCounters), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = createSharedCounters(resourceapi.ResourceSliceMaxSharedCounters + 1) + return slice + }(), + }, + "missing-sharedcounter-consumes-counter": { + wantFailures: field.ErrorList{ + field.Required(field.NewPath("spec", "devices").Index(0).Child("basic", "consumesCounter").Index(0).Child("sharedCounter"), ""), + field.Invalid(field.NewPath("spec", "devices").Index(0).Child("basic", "consumesCounter").Index(0).Child("sharedCounter"), "", "must reference a counterSet defined in the ResourceSlice sharedCounters"), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = createSharedCounters(1) + slice.Spec.Devices[0].Basic.ConsumesCounter = []resourceapi.DeviceCounterConsumption{ + { + Counters: testCounters(), + }, + } + return slice + }(), + }, + "missing-counter-consumes-counter": { + wantFailures: field.ErrorList{ + field.Required(field.NewPath("spec", "devices").Index(0).Child("basic", "consumesCounter").Index(0).Child("counters"), ""), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = createSharedCounters(1) + slice.Spec.Devices[0].Basic.ConsumesCounter = []resourceapi.DeviceCounterConsumption{ + { + SharedCounter: "sharedcounters-0", + }, + } + return slice + }(), + }, + "wrong-counterref-consumes-counter": { + wantFailures: field.ErrorList{ + field.Invalid(field.NewPath("spec", "devices").Index(0).Child("basic", "consumesCounter").Index(0).Child("counters"), "fake", "must reference a counter defined in the ResourceSlice sharedCounters"), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = createSharedCounters(1) + slice.Spec.Devices[0].Basic.ConsumesCounter = []resourceapi.DeviceCounterConsumption{ + { + Counters: map[string]resourceapi.Counter{ + "fake": {Value: resource.MustParse("1Gi")}, + }, + SharedCounter: "sharedcounters-0", + }, + } + return slice + }(), + }, + "wrong-sharedcounterref-consumes-counter": { + wantFailures: field.ErrorList{ + field.Invalid(field.NewPath("spec", "devices").Index(0).Child("basic", "consumesCounter").Index(0).Child("sharedCounter"), "fake", "must reference a counterSet defined in the ResourceSlice sharedCounters"), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = createSharedCounters(1) + slice.Spec.Devices[0].Basic.ConsumesCounter = []resourceapi.DeviceCounterConsumption{ + { + Counters: testCounters(), + SharedCounter: "fake", + }, + } + return slice + }(), + }, + "too-large-consumes-counter": { + wantFailures: field.ErrorList{ + field.TooMany(field.NewPath("spec", "devices").Index(0).Child("basic", "consumesCounter"), resourceapi.ResourceSliceMaxDeviceCounterConsumptions+1, resourceapi.ResourceSliceMaxSharedCounters), + field.TooMany(field.NewPath("spec", "sharedCounters"), resourceapi.ResourceSliceMaxDeviceCounterConsumptions+1, resourceapi.ResourceSliceMaxSharedCounters), + }, + slice: func() *resourceapi.ResourceSlice { + slice := testResourceSlice(goodName, goodName, driverName, 1) + slice.Spec.SharedCounters = createSharedCounters(resourceapi.ResourceSliceMaxDeviceCounterConsumptions + 1) + slice.Spec.Devices[0].Basic.ConsumesCounter = createConsumesCounter(resourceapi.ResourceSliceMaxDeviceCounterConsumptions + 1) + return slice + }(), + }, } for name, scenario := range scenarios { @@ -582,3 +815,25 @@ func TestValidateResourceSliceUpdate(t *testing.T) { }) } } + +func createSharedCounters(count int) []resourceapi.CounterSet { + sharedCounters := make([]resourceapi.CounterSet, count) + for i := 0; i < count; i++ { + sharedCounters[i] = resourceapi.CounterSet{ + Name: fmt.Sprintf("sharedcounters-%d", i), + Counters: testCounters(), + } + } + return sharedCounters +} + +func createConsumesCounter(count int) []resourceapi.DeviceCounterConsumption { + consumeCapacity := make([]resourceapi.DeviceCounterConsumption, count) + for i := 0; i < count; i++ { + consumeCapacity[i] = resourceapi.DeviceCounterConsumption{ + SharedCounter: fmt.Sprintf("sharedcounters-%d", i), + Counters: testCounters(), + } + } + return consumeCapacity +} diff --git a/pkg/registry/resource/resourceslice/strategy.go b/pkg/registry/resource/resourceslice/strategy.go index 6170aaac6bf..c6f0a8c2f36 100644 --- a/pkg/registry/resource/resourceslice/strategy.go +++ b/pkg/registry/resource/resourceslice/strategy.go @@ -157,6 +157,7 @@ func toSelectableFields(slice *resource.ResourceSlice) fields.Set { // dropDisabledFields removes fields which are covered by a feature gate. func dropDisabledFields(newSlice, oldSlice *resource.ResourceSlice) { dropDisabledDRADeviceTaintsFields(newSlice, oldSlice) + dropDisabledDRAPartitionableDevicesFields(newSlice, oldSlice) } func dropDisabledDRADeviceTaintsFields(newSlice, oldSlice *resource.ResourceSlice) { @@ -183,3 +184,45 @@ func draDeviceTaintsFeatureInUse(slice *resource.ResourceSlice) bool { } return false } + +func dropDisabledDRAPartitionableDevicesFields(newSlice, oldSlice *resource.ResourceSlice) { + if utilfeature.DefaultFeatureGate.Enabled(features.DRAPartitionableDevices) || draPartitionableDevicesFeatureInUse(oldSlice) { + return + } + + newSlice.Spec.SharedCounters = nil + newSlice.Spec.PerDeviceNodeSelection = nil + for _, device := range newSlice.Spec.Devices { + if device.Basic != nil { + device.Basic.ConsumesCounter = nil + device.Basic.NodeName = nil + device.Basic.NodeSelector = nil + device.Basic.AllNodes = nil + } + + } +} + +func draPartitionableDevicesFeatureInUse(slice *resource.ResourceSlice) bool { + if slice == nil { + return false + } + + spec := slice.Spec + if len(spec.SharedCounters) > 0 || spec.PerDeviceNodeSelection != nil { + return true + } + + for _, device := range spec.Devices { + if device.Basic != nil { + if len(device.Basic.ConsumesCounter) > 0 { + return true + } + if device.Basic.NodeName != nil || device.Basic.NodeSelector != nil || device.Basic.AllNodes != nil { + return true + } + } + + } + return false +} diff --git a/pkg/registry/resource/resourceslice/strategy_test.go b/pkg/registry/resource/resourceslice/strategy_test.go index 4424f9fdeed..e84bb54dbc1 100644 --- a/pkg/registry/resource/resourceslice/strategy_test.go +++ b/pkg/registry/resource/resourceslice/strategy_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/assert" + k8sresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -31,7 +32,7 @@ import ( var slice = &resource.ResourceSlice{ ObjectMeta: metav1.ObjectMeta{ - Name: "valid-class", + Name: "valid-resource-slice", }, Spec: resource.ResourceSliceSpec{ NodeName: "valid-node-name", @@ -56,6 +57,68 @@ var sliceWithDeviceTaints = func() *resource.ResourceSlice { return slice }() +var sliceWithPartitionableDevices = &resource.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "valid-resource-slice", + }, + Spec: resource.ResourceSliceSpec{ + PerDeviceNodeSelection: func() *bool { + r := true + return &r + }(), + Driver: "testdriver.example.com", + Pool: resource.ResourcePool{ + Name: "valid-pool-name", + ResourceSliceCount: 1, + Generation: 1, + }, + SharedCounters: []resource.CounterSet{ + { + Name: "pool-1", + Counters: map[string]resource.Counter{ + "memory": { + Value: k8sresource.MustParse("40Gi"), + }, + }, + }, + }, + Devices: []resource.Device{ + { + Name: "device", + Basic: &resource.BasicDevice{ + ConsumesCounter: []resource.DeviceCounterConsumption{ + { + SharedCounter: "pool-1", + Counters: map[string]resource.Counter{ + "memory": { + Value: k8sresource.MustParse("40Gi"), + }, + }, + }, + }, + NodeName: func() *string { + r := "valid-node-name" + return &r + }(), + Attributes: map[resource.QualifiedName]resource.DeviceAttribute{ + resource.QualifiedName("version"): { + StringValue: func() *string { + v := "v1" + return &v + }(), + }, + }, + Capacity: map[resource.QualifiedName]resource.DeviceCapacity{ + resource.QualifiedName("memory"): { + Value: k8sresource.MustParse("40Gi"), + }, + }, + }, + }, + }, + }, +} + func TestResourceSliceStrategy(t *testing.T) { if Strategy.NamespaceScoped() { t.Errorf("ResourceSlice must not be namespace scoped") @@ -70,6 +133,7 @@ func TestResourceSliceStrategyCreate(t *testing.T) { testCases := map[string]struct { obj *resource.ResourceSlice deviceTaints bool + partitionableDevices bool expectedValidationError bool expectObj *resource.ResourceSlice }{ @@ -107,11 +171,53 @@ func TestResourceSliceStrategyCreate(t *testing.T) { return obj }(), }, + "drop-fields-partitionable-devices": { + obj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.Spec.PerDeviceNodeSelection = func() *bool { + r := false + return &r + }() + obj.Spec.NodeName = "valid-node-name" + return obj + }(), + partitionableDevices: false, + expectObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ObjectMeta.Generation = 1 + obj.Spec.SharedCounters = nil + obj.Spec.PerDeviceNodeSelection = nil + obj.Spec.NodeName = "valid-node-name" + obj.Spec.Devices[0].Basic.NodeName = nil + obj.Spec.Devices[0].Basic.NodeSelector = nil + obj.Spec.Devices[0].Basic.AllNodes = nil + obj.Spec.Devices[0].Basic.ConsumesCounter = nil + return obj + }(), + }, + // This should return a validation error since the slice will not + // have a node selector after the perDeviceNodeSelection field got + // dropped. + "drop-fields-partitionable-devices-with-per-device-node-selection": { + obj: sliceWithPartitionableDevices, + partitionableDevices: false, + expectedValidationError: true, + }, + "keep-fields-partitionable-devices": { + obj: sliceWithPartitionableDevices, + partitionableDevices: true, + expectObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ObjectMeta.Generation = 1 + return obj + }(), + }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRADeviceTaints, tc.deviceTaints) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAPartitionableDevices, tc.partitionableDevices) obj := tc.obj.DeepCopy() @@ -138,6 +244,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { oldObj *resource.ResourceSlice newObj *resource.ResourceSlice deviceTaints bool + partitionableDevices bool expectValidationError bool expectObj *resource.ResourceSlice }{ @@ -221,11 +328,96 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { return obj }(), }, + "drop-fields-partitionable-devices": { + oldObj: slice, + newObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + obj.Spec.PerDeviceNodeSelection = func() *bool { + r := false + return &r + }() + obj.Spec.NodeName = "valid-node-name" + return obj + }(), + partitionableDevices: false, + expectObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + obj.Generation = 1 + obj.Spec.SharedCounters = nil + obj.Spec.PerDeviceNodeSelection = nil + obj.Spec.NodeName = "valid-node-name" + obj.Spec.Devices[0].Basic.ConsumesCounter = nil + obj.Spec.Devices[0].Basic.NodeName = nil + return obj + }(), + }, + "drop-fields-partitionable-devices-with-per-device-node-selection": { + oldObj: slice, + newObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + return obj + }(), + partitionableDevices: false, + expectValidationError: true, + }, + "keep-fields-partitionable-devices": { + oldObj: slice, + newObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + obj.Spec.NodeName = "valid-node-name" + obj.Spec.PerDeviceNodeSelection = nil + obj.Spec.Devices[0].Basic.NodeName = nil + return obj + }(), + partitionableDevices: true, + expectObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + obj.Generation = 1 + obj.Spec.NodeName = "valid-node-name" + obj.Spec.PerDeviceNodeSelection = nil + obj.Spec.Devices[0].Basic.NodeName = nil + return obj + }(), + }, + "keep-existing-fields-partitionable-devices": { + oldObj: sliceWithPartitionableDevices, + newObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + return obj + }(), + partitionableDevices: true, + expectObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + return obj + }(), + }, + "keep-existing-fields-partitionable-devices-disabled-feature": { + oldObj: sliceWithPartitionableDevices, + newObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + return obj + }(), + partitionableDevices: false, + expectObj: func() *resource.ResourceSlice { + obj := sliceWithPartitionableDevices.DeepCopy() + obj.ResourceVersion = "4" + return obj + }(), + }, } for name, tc := range testcases { t.Run(name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRADeviceTaints, tc.deviceTaints) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAPartitionableDevices, tc.partitionableDevices) oldObj := tc.oldObj.DeepCopy() newObj := tc.newObj.DeepCopy() diff --git a/staging/src/k8s.io/api/resource/v1alpha3/types.go b/staging/src/k8s.io/api/resource/v1alpha3/types.go index ffb82ed8e55..1981b6e9774 100644 --- a/staging/src/k8s.io/api/resource/v1alpha3/types.go +++ b/staging/src/k8s.io/api/resource/v1alpha3/types.go @@ -110,7 +110,7 @@ type ResourceSliceSpec struct { // new nodes of the same type as some old node might also make new // resources available. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // This field is immutable. // // +optional @@ -122,7 +122,7 @@ type ResourceSliceSpec struct { // // Must use exactly one term. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // // +optional // +oneOf=NodeSelection @@ -130,7 +130,7 @@ type ResourceSliceSpec struct { // AllNodes indicates that all nodes have access to the resources in the pool. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // // +optional // +oneOf=NodeSelection @@ -143,6 +143,66 @@ type ResourceSliceSpec struct { // +optional // +listType=atomic Devices []Device `json:"devices" protobuf:"bytes,6,name=devices"` + + // PerDeviceNodeSelection defines whether the access from nodes to + // resources in the pool is set on the ResourceSlice level or on each + // device. If it is set to true, every device defined the ResourceSlice + // must specify this individually. + // + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. + // + // +optional + // +oneOf=NodeSelection + // +featureGate=DRAPartitionableDevices + PerDeviceNodeSelection *bool `json:"perDeviceNodeSelection,omitempty" protobuf:"bytes,7,name=perDeviceNodeSelection"` + + // SharedCounters defines a list of counter sets, each of which + // has a name and a list of counters available. + // + // The names of the SharedCounters must be unique in the ResourceSlice. + // + // The maximum number of SharedCounters is 32. + // + // +optional + // +listType=atomic + // +featureGate=DRAPartitionableDevices + SharedCounters []CounterSet `json:"sharedCounters,omitempty" protobuf:"bytes,8,name=sharedCounters"` +} + +// CounterSet defines a named set of counters +// that are available to be used by devices defined in the +// ResourceSlice. +// +// The counters are not allocatable by themselves, but +// can be referenced by devices. When a device is allocated, +// the portion of counters it uses will no longer be available for use +// by other devices. +type CounterSet struct { + // Name defines the name of the counter set. + // It must be a DNS label. + // + // +required + Name string `json:"name" protobuf:"bytes,1,name=name"` + + // Counters defines the set of counters for this CounterSet + // The name of each counter must be unique in that set and must be a DNS label. + // + // To ensure this uniqueness, capacities defined by the vendor + // must be listed without the driver name as domain prefix in + // their name. All others must be listed with their domain prefix. + // + // The maximum number of counters is 32. + // + // +required + Counters map[string]Counter `json:"counters,omitempty" protobuf:"bytes,2,name=counters"` +} + +// Counter describes a quantity associated with a device. +type Counter struct { + // Value defines how much of a certain device counter is available. + // + // +required + Value resource.Quantity `json:"value" protobuf:"bytes,1,rep,name=value"` } // DriverNameMaxLength is the maximum valid length of a driver name in the @@ -224,6 +284,52 @@ type BasicDevice struct { // +optional Capacity map[QualifiedName]resource.Quantity `json:"capacity,omitempty" protobuf:"bytes,2,rep,name=capacity"` + // ConsumesCounter defines a list of references to sharedCounters + // and the set of counters that the device will + // consume from those counter sets. + // + // There can only be a single entry per counterSet. + // + // The maximum number of device counter consumption entries + // is 32. This is the same as the maximum number of shared counters + // allowed in a ResourceSlice. + // + // +optional + // +listType=atomic + // +featureGate=DRAPartitionableDevices + ConsumesCounter []DeviceCounterConsumption `json:"consumesCounter,omitempty" protobuf:"bytes,3,rep,name=consumesCounter"` + + // NodeName identifies the node where the device is available. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + // +featureGate=DRAPartitionableDevices + NodeName *string `json:"nodeName,omitempty" protobuf:"bytes,4,opt,name=nodeName"` + + // NodeSelector defines the nodes where the device is available. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + // +featureGate=DRAPartitionableDevices + NodeSelector *v1.NodeSelector `json:"nodeSelector,omitempty" protobuf:"bytes,5,opt,name=nodeSelector"` + + // AllNodes indicates that all nodes have access to the device. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + // +featureGate=DRAPartitionableDevices + AllNodes *bool `json:"allNodes,omitempty" protobuf:"bytes,6,opt,name=allNodes"` + + // If specified, these are the driver-defined taints. // // The maximum number of taints is 8. @@ -234,7 +340,26 @@ type BasicDevice struct { // +optional // +listType=atomic // +featureGate=DRADeviceTaints - Taints []DeviceTaint `json:"taints,omitempty" protobuf:"bytes,3,rep,name=taints"` + Taints []DeviceTaint `json:"taints,omitempty" protobuf:"bytes,7,rep,name=taints"` +} + +// DeviceCounterConsumption defines a set of counters that +// a device will consume from a CounterSet. +type DeviceCounterConsumption struct { + // SharedCounter defines the shared counter from which the + // counters defined will be consumed. + // + // +required + SharedCounter string `json:"sharedCounter" protobuf:"bytes,1,opt,name=sharedCounter"` + + // Counters defines the Counter that will be consumed by + // the device. + // + // + // The maximum number of Counters is 32. + // + // +required + Counters map[string]Counter `json:"counters,omitempty" protobuf:"bytes,2,opt,name=counters"` } // Limit for the sum of the number of entries in both attributes and capacity. diff --git a/staging/src/k8s.io/api/resource/v1beta1/types.go b/staging/src/k8s.io/api/resource/v1beta1/types.go index 36d3d45c2e9..41903dc510b 100644 --- a/staging/src/k8s.io/api/resource/v1beta1/types.go +++ b/staging/src/k8s.io/api/resource/v1beta1/types.go @@ -109,7 +109,7 @@ type ResourceSliceSpec struct { // new nodes of the same type as some old node might also make new // resources available. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // This field is immutable. // // +optional @@ -121,7 +121,7 @@ type ResourceSliceSpec struct { // // Must use exactly one term. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // // +optional // +oneOf=NodeSelection @@ -129,7 +129,7 @@ type ResourceSliceSpec struct { // AllNodes indicates that all nodes have access to the resources in the pool. // - // Exactly one of NodeName, NodeSelector and AllNodes must be set. + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. // // +optional // +oneOf=NodeSelection @@ -142,6 +142,62 @@ type ResourceSliceSpec struct { // +optional // +listType=atomic Devices []Device `json:"devices" protobuf:"bytes,6,name=devices"` + + // PerDeviceNodeSelection defines whether the access from nodes to + // resources in the pool is set on the ResourceSlice level or on each + // device. If it is set to true, every device defined the ResourceSlice + // must specify this individually. + // + // Exactly one of NodeName, NodeSelector, AllNodes, and PerDeviceNodeSelection must be set. + // + // +optional + // +oneOf=NodeSelection + // +featureGate=DRAPartitionableDevices + PerDeviceNodeSelection *bool `json:"perDeviceNodeSelection,omitempty" protobuf:"bytes,7,name=perDeviceNodeSelection"` + + // SharedCounters defines a list of counter sets, each of which + // has a name and a list of counters available. + // + // The names of the SharedCounters must be unique in the ResourceSlice. + // + // The maximum number of SharedCounters is 32. + // + // +optional + // +listType=atomic + // +featureGate=DRAPartitionableDevices + SharedCounters []CounterSet `json:"sharedCounters,omitempty" protobuf:"bytes,8,name=sharedCounters"` +} + +// CounterSet defines a named set of counters +// that are available to be used by devices defined in the +// ResourceSlice. +// +// The counters are not allocatable by themselves, but +// can be referenced by devices. When a device is allocated, +// the portion of counters it uses will no longer be available for use +// by other devices. +type CounterSet struct { + // Name defines the name of the counter set. + // It must be a DNS label. + // + // +required + Name string `json:"name" protobuf:"bytes,1,name=name"` + + // Counters defines the set of counters for this CounterSet + // The name of each counter must be unique in that set and must be a DNS label. + // + // The maximum number of counters is 32. + // + // +required + Counters map[string]Counter `json:"counters,omitempty" protobuf:"bytes,2,name=counters"` +} + +// Counter describes a quantity associated with a device. +type Counter struct { + // Value defines how much of a certain device counter is available. + // + // +required + Value resource.Quantity `json:"value" protobuf:"bytes,1,rep,name=value"` } // DriverNameMaxLength is the maximum valid length of a driver name in the @@ -223,6 +279,50 @@ type BasicDevice struct { // +optional Capacity map[QualifiedName]DeviceCapacity `json:"capacity,omitempty" protobuf:"bytes,2,rep,name=capacity"` + // ConsumesCounter defines a list of references to sharedCounters + // and the set of counters that the device will + // consume from those counter sets. + // + // There can only be a single entry per counterSet. + // + // The maximum number of device counter consumption entries + // is 32. This is the same as the maximum number of shared counters + // allowed in a ResourceSlice. + // + // +optional + // +listType=atomic + // +featureGate=DRAPartitionableDevices + ConsumesCounter []DeviceCounterConsumption `json:"consumesCounter,omitempty" protobuf:"bytes,3,rep,name=consumesCounter"` + + // NodeName identifies the node where the device is available. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + // +featureGate=DRAPartitionableDevices + NodeName *string `json:"nodeName,omitempty" protobuf:"bytes,4,opt,name=nodeName"` + + // NodeSelector defines the nodes where the device is available. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + NodeSelector *v1.NodeSelector `json:"nodeSelector,omitempty" protobuf:"bytes,5,opt,name=nodeSelector"` + + // AllNodes indicates that all nodes have access to the device. + // + // Must only be set if Spec.PerDeviceNodeSelection is set to true. + // At most one of NodeName, NodeSelector and AllNodes can be set. + // + // +optional + // +oneOf=DeviceNodeSelection + // +featureGate=DRAPartitionableDevices + AllNodes *bool `json:"allNodes,omitempty" protobuf:"bytes,6,opt,name=allNodes"` + // If specified, these are the driver-defined taints. // // The maximum number of taints is 8. @@ -233,7 +333,26 @@ type BasicDevice struct { // +optional // +listType=atomic // +featureGate=DRADeviceTaints - Taints []DeviceTaint `json:"taints,omitempty" protobuf:"bytes,3,rep,name=taints"` + Taints []DeviceTaint `json:"taints,omitempty" protobuf:"bytes,7,rep,name=taints"` +} + +// DeviceCounterConsumption defines a set of counters that +// a device will consume from a CounterSet. +type DeviceCounterConsumption struct { + // SharedCounter defines the shared counter from which the + // counters defined will be consumed. + // + // +required + SharedCounter string `json:"sharedCounter" protobuf:"bytes,1,opt,name=sharedCounter"` + + // Counters defines the Counter that will be consumed by + // the device. + // + // + // The maximum number of Counters is 32. + // + // +required + Counters map[string]Counter `json:"counters,omitempty" protobuf:"bytes,2,opt,name=counters"` } // DeviceCapacity describes a quantity associated with a device. diff --git a/staging/src/k8s.io/dynamic-resource-allocation/api/types.go b/staging/src/k8s.io/dynamic-resource-allocation/api/types.go index c07d683691d..ad62dc7f7b0 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/api/types.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/api/types.go @@ -30,12 +30,19 @@ type ResourceSlice struct { } type ResourceSliceSpec struct { - Driver UniqueString - Pool ResourcePool - NodeName UniqueString - NodeSelector *v1.NodeSelector - AllNodes bool - Devices []Device + Driver UniqueString + Pool ResourcePool + NodeName UniqueString + NodeSelector *v1.NodeSelector + AllNodes bool + Devices []Device + PerDeviceNodeSelection *bool + SharedCounters []CounterSet +} + +type CounterSet struct { + Name UniqueString + Counters map[string]Counter } type ResourcePool struct { @@ -49,11 +56,20 @@ type Device struct { } type BasicDevice struct { - Attributes map[QualifiedName]DeviceAttribute - Capacity map[QualifiedName]DeviceCapacity + Attributes map[QualifiedName]DeviceAttribute + Capacity map[QualifiedName]DeviceCapacity + ConsumesCounter []DeviceCounterConsumption + NodeName *string + NodeSelector *v1.NodeSelector + AllNodes *bool Taints []resourceapi.DeviceTaint } +type DeviceCounterConsumption struct { + SharedCounter UniqueString + Counters map[string]Counter +} + type QualifiedName string type FullyQualifiedName string @@ -69,6 +85,10 @@ type DeviceCapacity struct { Value resource.Quantity } +type Counter struct { + Value resource.Quantity +} + type DeviceTaint struct { Key string Value string