From bf7dbb129fbd9f2ab8baf0b96d6aa8329816af21 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Tue, 7 Oct 2025 14:40:04 +0300 Subject: [PATCH] DRA: test performance for scoring extended resources --- .../scoring/extendedresource_scoring_test.go | 57 +++++++++++++ .../scoring/performance-config.yaml | 84 +++++++++++++++++++ .../scoring/scheduler-config.yaml | 16 ++++ ...deviceclass-extended-resource-scoring.yaml | 11 +++ .../pod-with-extended-resource-scoring.yaml | 15 ++++ 5 files changed, 183 insertions(+) create mode 100644 test/integration/scheduler_perf/dra/extendedresource/scoring/extendedresource_scoring_test.go create mode 100644 test/integration/scheduler_perf/dra/extendedresource/scoring/performance-config.yaml create mode 100644 test/integration/scheduler_perf/dra/extendedresource/scoring/scheduler-config.yaml create mode 100644 test/integration/scheduler_perf/dra/templates/deviceclass-extended-resource-scoring.yaml create mode 100644 test/integration/scheduler_perf/dra/templates/pod-with-extended-resource-scoring.yaml diff --git a/test/integration/scheduler_perf/dra/extendedresource/scoring/extendedresource_scoring_test.go b/test/integration/scheduler_perf/dra/extendedresource/scoring/extendedresource_scoring_test.go new file mode 100644 index 00000000000..7a6c7c68153 --- /dev/null +++ b/test/integration/scheduler_perf/dra/extendedresource/scoring/extendedresource_scoring_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extendedresourcescoring + +import ( + "fmt" + "os" + "testing" + + _ "k8s.io/component-base/logs/json/register" + "k8s.io/dynamic-resource-allocation/structured" + perf "k8s.io/kubernetes/test/integration/scheduler_perf" +) + +func TestMain(m *testing.M) { + if err := perf.InitTests(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + m.Run() +} + +func TestSchedulerPerf(t *testing.T) { + // Verify correct behavior with all available allocators for this feature. + for _, allocatorName := range []string{"experimental"} { + t.Run(allocatorName, func(t *testing.T) { + structured.EnableAllocators(allocatorName) + defer structured.EnableAllocators() + + perf.RunIntegrationPerfScheduling(t, "performance-config.yaml") + }) + } +} + +func BenchmarkPerfScheduling(b *testing.B) { + // Restrict benchmarking to the allocator for this feature + // to ensure that we do not accidentally pick something else. + structured.EnableAllocators("experimental") + defer structured.EnableAllocators() + + perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "dra_extendedresource_scoring", nil) +} diff --git a/test/integration/scheduler_perf/dra/extendedresource/scoring/performance-config.yaml b/test/integration/scheduler_perf/dra/extendedresource/scoring/performance-config.yaml new file mode 100644 index 00000000000..4308f23d690 --- /dev/null +++ b/test/integration/scheduler_perf/dra/extendedresource/scoring/performance-config.yaml @@ -0,0 +1,84 @@ +# Copyright 2025 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration specifically for testing DRA extended resource scoring performance. +# This config ensures that multiple nodes can satisfy each pod's extended resource +# requirements, forcing the scheduler to use scoring to choose between nodes. + + +# ExtendedResourceScoring uses pods with extended resources that can be satisfied +# by multiple nodes, triggering scoring competition to test the DRA scoring implementation. +- name: ExtendedResourceScoring + featureGates: + DynamicResourceAllocation: true + DRAExtendedResource: true + schedulerConfigPath: scheduler-config.yaml + workloadTemplate: + - opcode: createNodes + countParam: $nodesWithoutDRA + - opcode: createNodes + nodeTemplatePath: ../../templates/node-with-dra-test-driver.yaml + countParam: $nodesWithDRA + - opcode: createResourceDriver + driverName: test-driver.cdi.k8s.io + nodes: scheduler-perf-dra-* + maxClaimsPerNodeParam: $maxClaimsPerNode + - opcode: createAny + templatePath: ../../templates/deviceclass-extended-resource-scoring.yaml + countParam: $classes + - opcode: createPods + namespace: init + countParam: $initPods + podTemplatePath: ../../templates/pod-with-extended-resource-scoring.yaml + - opcode: createPods + namespace: test + countParam: $measurePods + podTemplatePath: ../../templates/pod-with-extended-resource-scoring.yaml + collectMetrics: true + workloads: + - name: fast + featureGates: + DRAExtendedResource: true + labels: [integration-test, short] + params: + # Use fewer classes than nodes to ensure multiple nodes can satisfy + # each pod's extended resource requirements, forcing scoring competition. + classes: 2 + nodesWithDRA: 3 + nodesWithoutDRA: 1 + initPods: 0 + measurePods: 10 + maxClaimsPerNode: 10 + - name: 2000pods_200nodes + featureGates: + DRAExtendedResource: true + labels: [integration-test, performance] + params: + classes: 10 + nodesWithDRA: 200 + nodesWithoutDRA: 0 + initPods: 0 + measurePods: 2000 + maxClaimsPerNode: 20 + - name: 5000pods_500nodes + featureGates: + DRAExtendedResource: true + labels: [integration-test, performance] + params: + classes: 20 + nodesWithDRA: 500 + nodesWithoutDRA: 0 + initPods: 0 + measurePods: 5000 + maxClaimsPerNode: 20 diff --git a/test/integration/scheduler_perf/dra/extendedresource/scoring/scheduler-config.yaml b/test/integration/scheduler_perf/dra/extendedresource/scoring/scheduler-config.yaml new file mode 100644 index 00000000000..e034385d9a3 --- /dev/null +++ b/test/integration/scheduler_perf/dra/extendedresource/scoring/scheduler-config.yaml @@ -0,0 +1,16 @@ +apiVersion: kubescheduler.config.k8s.io/v1 +kind: KubeSchedulerConfiguration +profiles: +- schedulerName: default-scheduler + pluginConfig: + - name: NodeResourcesFit + args: + scoringStrategy: + type: LeastAllocated + resources: + - name: cpu + weight: 1 + - name: memory + weight: 1 + - name: example.com/gpu + weight: 1 diff --git a/test/integration/scheduler_perf/dra/templates/deviceclass-extended-resource-scoring.yaml b/test/integration/scheduler_perf/dra/templates/deviceclass-extended-resource-scoring.yaml new file mode 100644 index 00000000000..114651606bd --- /dev/null +++ b/test/integration/scheduler_perf/dra/templates/deviceclass-extended-resource-scoring.yaml @@ -0,0 +1,11 @@ +apiVersion: resource.k8s.io/v1beta2 +kind: DeviceClass +metadata: + name: scoring-test-class-{{.Index}} +spec: + selectors: + - cel: + expression: device.driver == "test-driver.cdi.k8s.io" + # All device classes expose the same extended resource name + # This allows multiple nodes to satisfy the same pod's extended resource request + extendedResourceName: example.com/gpu diff --git a/test/integration/scheduler_perf/dra/templates/pod-with-extended-resource-scoring.yaml b/test/integration/scheduler_perf/dra/templates/pod-with-extended-resource-scoring.yaml new file mode 100644 index 00000000000..11c301414a3 --- /dev/null +++ b/test/integration/scheduler_perf/dra/templates/pod-with-extended-resource-scoring.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: scoring-test-pod-{{.Index}} +spec: + containers: + - image: registry.k8s.io/pause:3.9 + name: pause + resources: + requests: + # All pods request the same extended resource + # This forces the scheduler to choose between multiple nodes + example.com/gpu: 1 + limits: + example.com/gpu: 1