diff --git a/test/integration/dra/dra_test.go b/test/integration/dra/dra_test.go index eb64ebeeed2..866794c5a02 100644 --- a/test/integration/dra/dra_test.go +++ b/test/integration/dra/dra_test.go @@ -47,6 +47,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" resourceapiac "k8s.io/client-go/applyconfigurations/resource/v1" @@ -842,12 +843,16 @@ func testPublishResourceSlices(tCtx ktesting.TContext, haveLatestAPI bool, disab }, }, { - SharedCounters: []resourceapi.CounterSet{{ - Name: "gpu-0", - Counters: map[string]resourceapi.Counter{ - "mem": {Value: resource.MustParse("1")}, + SharedCounters: []resourceapi.CounterSet{ + { + Name: "gpu-0", + Counters: map[string]resourceapi.Counter{ + "mem": {Value: resource.MustParse("1")}, + }, }, - }}, + }, + }, + { Devices: []resourceapi.Device{ { Name: "device-tainted-default", @@ -910,32 +915,46 @@ func testPublishResourceSlices(tCtx ktesting.TContext, haveLatestAPI bool, disab Devices: sl.Devices, }) } - for _, disabled := range disabledFeatures { - switch disabled { - case features.DRADeviceTaints: - for i := range expectedSliceSpecs { - for e := range expectedSliceSpecs[i].Devices { - expectedSliceSpecs[i].Devices[e].Taints = nil + + // Keep track of the disabled features used in each slice. This allows us to check + // that we get the right set of features listed in the droppedFields error. + disabledFeaturesBySlice := make([]sets.Set[featuregate.Feature], len(resources.Pools[poolName].Slices)) + for i := range expectedSliceSpecs { + disabledFeaturesForSlice := sets.New[featuregate.Feature]() + for _, disabled := range disabledFeatures { + switch disabled { + case features.DRADeviceTaints: + for e, device := range expectedSliceSpecs[i].Devices { + if device.Taints != nil { + expectedSliceSpecs[i].Devices[e].Taints = nil + disabledFeaturesForSlice.Insert(disabled) + } } - } - case features.DRAPartitionableDevices: - for i := range expectedSliceSpecs { - expectedSliceSpecs[i].SharedCounters = nil - for e := range expectedSliceSpecs[i].Devices { - expectedSliceSpecs[i].Devices[e].ConsumesCounters = nil + case features.DRAPartitionableDevices: + if expectedSliceSpecs[i].SharedCounters != nil { + expectedSliceSpecs[i].SharedCounters = nil + disabledFeaturesForSlice.Insert(disabled) } - } - case features.DRADeviceBindingConditions: - for i := range expectedSliceSpecs { - for e := range expectedSliceSpecs[i].Devices { - expectedSliceSpecs[i].Devices[e].BindingConditions = nil - expectedSliceSpecs[i].Devices[e].BindingFailureConditions = nil - expectedSliceSpecs[i].Devices[e].BindsToNode = nil + for e, device := range expectedSliceSpecs[i].Devices { + if device.ConsumesCounters != nil { + expectedSliceSpecs[i].Devices[e].ConsumesCounters = nil + disabledFeaturesForSlice.Insert(disabled) + } } + case features.DRADeviceBindingConditions: + for e, device := range expectedSliceSpecs[i].Devices { + if device.BindingConditions != nil || device.BindingFailureConditions != nil || device.BindsToNode != nil { + expectedSliceSpecs[i].Devices[e].BindingConditions = nil + expectedSliceSpecs[i].Devices[e].BindingFailureConditions = nil + expectedSliceSpecs[i].Devices[e].BindsToNode = nil + disabledFeaturesForSlice.Insert(disabled) + } + } + default: + tCtx.Fatalf("faulty test, case for %s missing", disabled) } - default: - tCtx.Fatalf("faulty test, case for %s missing", disabled) } + disabledFeaturesBySlice[i] = disabledFeaturesForSlice } var expectedSlices []any for _, spec := range expectedSliceSpecs { @@ -1050,10 +1069,13 @@ func testPublishResourceSlices(tCtx ktesting.TContext, haveLatestAPI bool, disab var droppedFields *resourceslice.DroppedFieldsError if errors.As(err, &droppedFields) { var disabled []string - for _, feature := range disabledFeatures { + for _, feature := range disabledFeaturesBySlice[droppedFields.SliceIndex].UnsortedList() { disabled = append(disabled, string(feature)) } - assert.ErrorContains(tCtx, err, fmt.Sprintf("pool %q, slice #1: some fields were dropped by the apiserver, probably because these features are disabled: %s", poolName, strings.Join(disabled, " "))) + // Make sure the error is about the right resource pool. + assert.Equal(tCtx, poolName, droppedFields.PoolName) + // Make sure the error identifies the correct disabled features. + assert.ElementsMatch(tCtx, disabled, droppedFields.DisabledFeatures()) gotDroppedFieldError.Store(true) } else if validationErrorsOkay.Load() && apierrors.IsInvalid(err) { gotValidationError.Store(true) @@ -1133,6 +1155,9 @@ func testPublishResourceSlices(tCtx ktesting.TContext, haveLatestAPI bool, disab slices, err := tCtx.Client().ResourceV1().ResourceSlices().List(tCtx, listDriverSlices) tCtx.ExpectNoError(err, "list slices") for _, slice := range slices.Items { + if len(slice.Spec.Devices) == 0 { + continue + } if slice.Spec.Devices[0].Attributes == nil { slice.Spec.Devices[0].Attributes = make(map[resourceapi.QualifiedName]resourceapi.DeviceAttribute) } diff --git a/test/integration/scheduler_perf/dra/partitionabledevices/performance-config.yaml b/test/integration/scheduler_perf/dra/partitionabledevices/performance-config.yaml index 3fdbd1b35c1..149881c3e9b 100644 --- a/test/integration/scheduler_perf/dra/partitionabledevices/performance-config.yaml +++ b/test/integration/scheduler_perf/dra/partitionabledevices/performance-config.yaml @@ -32,7 +32,10 @@ nodeTemplatePath: ../templates/node-with-dra-test-driver.yaml countParam: $nodesWithDRA - opcode: createAny - templatePath: ../templates/resourceslice-partitionable.yaml + templatePath: ../templates/resourceslice-partitionable-countersets.yaml + countParam: $resourceSlices + - opcode: createAny + templatePath: ../templates/resourceslice-partitionable-devices.yaml countParam: $resourceSlices - opcode: createAny templatePath: ../templates/deviceclass.yaml @@ -106,7 +109,10 @@ nodeTemplatePath: ../templates/node-with-dra-test-driver.yaml countParam: $nodesWithDRA - opcode: createAny - templatePath: ../templates/resourceslice-partitionable-backtracking.yaml + templatePath: ../templates/resourceslice-partitionable-backtracking-countersets.yaml + countParam: $resourceSlices + - opcode: createAny + templatePath: ../templates/resourceslice-partitionable-backtracking-devices.yaml countParam: $resourceSlices - opcode: createAny templatePath: ../templates/deviceclass.yaml diff --git a/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking-countersets.yaml b/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking-countersets.yaml new file mode 100644 index 00000000000..e08406f6a72 --- /dev/null +++ b/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking-countersets.yaml @@ -0,0 +1,40 @@ +kind: ResourceSlice +apiVersion: resource.k8s.io/v1 +metadata: + name: resourceslice-countersets-{{.Index}} +spec: + pool: + name: resourceslice-{{.Index}} + generation: 1 + resourceSliceCount: 2 + driver: test-driver.cdi.k8s.io + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: node-with-dra + operator: In + values: + - "true" + sharedCounters: + - name: counter-set-for-cpu + counters: + cpu1: + value: "1" + cpu2: + value: "1" + cpu3: + value: "1" + cpu4: + value: "1" + cpu5: + value: "1" + cpu6: + value: "1" + cpu7: + value: "1" + cpu8: + value: "1" + - name: counter-set-for-mem + counters: + mem: + value: "70Gi" diff --git a/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking.yaml b/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking-devices.yaml similarity index 82% rename from test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking.yaml rename to test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking-devices.yaml index 20a915b4210..47f55f7ca88 100644 --- a/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking.yaml +++ b/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-backtracking-devices.yaml @@ -1,12 +1,12 @@ kind: ResourceSlice apiVersion: resource.k8s.io/v1 metadata: - name: resourceslice-{{.Index}} + name: resourceslice-devices-{{.Index}} spec: pool: name: resourceslice-{{.Index}} generation: 1 - resourceSliceCount: 1 + resourceSliceCount: 2 driver: test-driver.cdi.k8s.io nodeSelector: nodeSelectorTerms: @@ -15,29 +15,6 @@ spec: operator: In values: - "true" - sharedCounters: - - name: counter-set-for-cpu - counters: - cpu1: - value: "1" - cpu2: - value: "1" - cpu3: - value: "1" - cpu4: - value: "1" - cpu5: - value: "1" - cpu6: - value: "1" - cpu7: - value: "1" - cpu8: - value: "1" - - name: counter-set-for-mem - counters: - mem: - value: "70Gi" devices: - name: device-1 consumesCounters: diff --git a/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-countersets.yaml b/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-countersets.yaml new file mode 100644 index 00000000000..3db481e5d64 --- /dev/null +++ b/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-countersets.yaml @@ -0,0 +1,28 @@ +kind: ResourceSlice +apiVersion: resource.k8s.io/v1 +metadata: + name: resourceslice-countersets-{{.Index}} +spec: + pool: + name: resourceslice-{{.Index}} + generation: 1 + resourceSliceCount: 2 + driver: test-driver.cdi.k8s.io + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: node-with-dra + operator: In + values: + - "true" + sharedCounters: + - name: counter-set + counters: + counter1: + value: "1" + counter2: + value: "1" + counter3: + value: "1" + counter4: + value: "1" diff --git a/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable.yaml b/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-devices.yaml similarity index 89% rename from test/integration/scheduler_perf/dra/templates/resourceslice-partitionable.yaml rename to test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-devices.yaml index db3abc68b38..bf9ee3e8c50 100644 --- a/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable.yaml +++ b/test/integration/scheduler_perf/dra/templates/resourceslice-partitionable-devices.yaml @@ -1,12 +1,12 @@ kind: ResourceSlice apiVersion: resource.k8s.io/v1 metadata: - name: resourceslice-{{.Index}} + name: resourceslice-devices-{{.Index}} spec: pool: name: resourceslice-{{.Index}} generation: 1 - resourceSliceCount: 1 + resourceSliceCount: 2 driver: test-driver.cdi.k8s.io nodeSelector: nodeSelectorTerms: @@ -15,17 +15,6 @@ spec: operator: In values: - "true" - sharedCounters: - - name: counter-set - counters: - counter1: - value: "1" - counter2: - value: "1" - counter3: - value: "1" - counter4: - value: "1" devices: # 2 counter devices - name: device-2-counters-1