From 6453fc2d030de1d79a81cf21968d01d7ad14f75d Mon Sep 17 00:00:00 2001 From: Praveen Krishna Date: Sun, 18 Jan 2026 21:36:36 +0000 Subject: [PATCH] Add Node Declared Features scheduler performance tests --- .../misc/performance-config.yaml | 9 +- .../node_declared_features_test.go | 187 ++++++++++++++++++ .../performance-config.yaml | 69 +++++++ .../templates/node-base.yaml | 17 ++ .../pod-with-pod-level-resources.yaml | 12 ++ 5 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 test/integration/scheduler_perf/nodedeclaredfeatures/node_declared_features_test.go create mode 100644 test/integration/scheduler_perf/nodedeclaredfeatures/performance-config.yaml create mode 100644 test/integration/scheduler_perf/nodedeclaredfeatures/templates/node-base.yaml create mode 100644 test/integration/scheduler_perf/nodedeclaredfeatures/templates/pod-with-pod-level-resources.yaml diff --git a/test/integration/scheduler_perf/misc/performance-config.yaml b/test/integration/scheduler_perf/misc/performance-config.yaml index a645035dfaa..df88633b5bf 100644 --- a/test/integration/scheduler_perf/misc/performance-config.yaml +++ b/test/integration/scheduler_perf/misc/performance-config.yaml @@ -100,7 +100,14 @@ initNodes: 5000 initPods: 5000 measurePods: 50000 - + - name: 5000Nodes_50000Pods_NodeDeclaredFeaturesDisabled + labels: [performance] + featureGates: + NodeDeclaredFeatures: false + params: + initNodes: 5000 + initPods: 5000 + measurePods: 50000 # This test case simulates the scheduling of daemonset. # https://github.com/kubernetes/kubernetes/issues/124709 - name: SchedulingDaemonset diff --git a/test/integration/scheduler_perf/nodedeclaredfeatures/node_declared_features_test.go b/test/integration/scheduler_perf/nodedeclaredfeatures/node_declared_features_test.go new file mode 100644 index 00000000000..7cc8f5c12ac --- /dev/null +++ b/test/integration/scheduler_perf/nodedeclaredfeatures/node_declared_features_test.go @@ -0,0 +1,187 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodedeclaredfeatures + +import ( + "context" + "fmt" + "os" + "reflect" + "sort" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" + _ "k8s.io/component-base/logs/json/register" + ndf "k8s.io/component-helpers/nodedeclaredfeatures" + ndffeatures "k8s.io/component-helpers/nodedeclaredfeatures/features" + "k8s.io/kubernetes/pkg/features" + perf "k8s.io/kubernetes/test/integration/scheduler_perf" + "k8s.io/kubernetes/test/utils/ktesting" +) + +// This test benchmarks the NodeDeclaredFeatures features performance impact. +// +// It mocks a variable number of node features registered (via 'numNodeDeclaredFeatures' param) but the +// inference function of all the registered features look for pod level resouces ('spec.resources'). +// The tests creates a pod with pod level resources to ensure that all features are inferred +// during scheduling. + +func TestMain(m *testing.M) { + if err := perf.InitTests(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + m.Run() +} + +// mockFeature is a mock implementation of the Feature interface for testing. +type mockFeature struct { + name string + maxVersion *version.Version +} + +func (f *mockFeature) Name() string { + return f.name +} + +func (f *mockFeature) Discover(cfg *ndf.NodeConfiguration) bool { + // This function is called by kubelet to discover features on the node and report them in the node status. + // In the test context, feature discovery is bypassed and we update node status directly after + // the node is created (see updateNodesWithDeclaredFeatures()), so this function is a no-op. + return true +} + +func (f *mockFeature) InferForScheduling(podInfo *ndf.PodInfo) bool { + // Check if the pod is using pod level resources. + return podInfo.Spec != nil && podInfo.Spec.Resources != nil +} + +func (f *mockFeature) InferForUpdate(oldPodInfo, newPodInfo *ndf.PodInfo) bool { + return false +} + +func (f *mockFeature) Requirements() *ndf.FeatureRequirements { + return nil +} + +func (f *mockFeature) MaxVersion() *version.Version { + return f.maxVersion +} + +func createMockFeature(name string, maxVersionStr string) ndf.Feature { + var v *version.Version + if maxVersionStr != "" { + v = version.MustParseSemantic(maxVersionStr) + } + return &mockFeature{ + name: name, + maxVersion: v, + } +} + +func setupFeatures(numNodeDeclaredFeatures int) func() { + nodeFeatures := make([]ndf.Feature, 0, numNodeDeclaredFeatures) + for i := 1; i <= numNodeDeclaredFeatures; i++ { + featureName := fmt.Sprintf("Feature%d", i) + nodeFeatures = append(nodeFeatures, createMockFeature(featureName, "")) + } + originalAllFeatures := ndffeatures.AllFeatures + ndffeatures.AllFeatures = nodeFeatures + return func() { + ndffeatures.AllFeatures = originalAllFeatures + } +} + +func preInitNodeDeclaredFeatures(t ktesting.TContext, w *perf.Workload) (func(), error) { + if !w.FeatureGates[features.NodeDeclaredFeatures] { + t.Logf("Skipping NodeDeclaredFeatures pre-init as the feature gate is disabled") + return func() {}, nil + } + + numNodeDeclaredFeatures, err := w.GetParam("numNodeDeclaredFeatures") + if err != nil { + t.Logf("numNodeDeclaredFeatures param not specified in workload config") + numNodeDeclaredFeatures = 0 + } + + t.Logf("PreInit: Setting up %d mock features for %s", numNodeDeclaredFeatures, w.Name) + // Setup mock features to be registered in the NDF library. + cleanupMockFeatures := setupFeatures(numNodeDeclaredFeatures) + + return cleanupMockFeatures, nil +} + +func updateNodesWithDeclaredFeatures(tCtx ktesting.TContext, w *perf.Workload, nodes *v1.NodeList) error { + if !w.FeatureGates[features.NodeDeclaredFeatures] { + return nil + } + + numNodeDeclaredFeatures, err := w.GetParam("numNodeDeclaredFeatures") + if err != nil { + tCtx.Logf("numNodeDeclaredFeatures param not specified in workload config") + numNodeDeclaredFeatures = 0 + } + + var featureNames []string + for i := 1; i <= numNodeDeclaredFeatures; i++ { + featureNames = append(featureNames, fmt.Sprintf("Feature%d", i)) + } + sort.Strings(featureNames) + + for _, node := range nodes.Items { + nodeToUpdate := node.DeepCopy() + nodeToUpdate.Status.DeclaredFeatures = featureNames + _, err := tCtx.Client().CoreV1().Nodes().UpdateStatus(tCtx.Context, nodeToUpdate, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update node %s status: %w", node.Name, err) + } + tCtx.Logf("Updated node %s status with %d features", node.Name, len(featureNames)) + } + + err = wait.PollUntilContextTimeout(tCtx.Context, 100*time.Millisecond, 60*time.Second, true, func(ctx context.Context) (bool, error) { + for _, n := range nodes.Items { + updatedNode, err := tCtx.Client().CoreV1().Nodes().Get(ctx, n.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + if !reflect.DeepEqual(updatedNode.Status.DeclaredFeatures, featureNames) { + return false, nil + } + } + return true, nil + }) + + if err != nil { + return fmt.Errorf("timeout waiting for all nodes to reflect status update: %w", err) + } + + return nil +} + +func TestSchedulerPerf(t *testing.T) { + perf.RunIntegrationPerfScheduling(t, "performance-config.yaml", perf.WithPreRunFn(preInitNodeDeclaredFeatures), perf.WithNodeUpdateFn(updateNodesWithDeclaredFeatures)) +} + +func BenchmarkPerfScheduling(b *testing.B) { + perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "nodedeclaredfeatures", nil, perf.WithPreRunFn(preInitNodeDeclaredFeatures), perf.WithNodeUpdateFn(updateNodesWithDeclaredFeatures)) +} diff --git a/test/integration/scheduler_perf/nodedeclaredfeatures/performance-config.yaml b/test/integration/scheduler_perf/nodedeclaredfeatures/performance-config.yaml new file mode 100644 index 00000000000..92d2a294af1 --- /dev/null +++ b/test/integration/scheduler_perf/nodedeclaredfeatures/performance-config.yaml @@ -0,0 +1,69 @@ +# The following labels are used in this file. (listed in ascending order of the number of covered test cases) +# +# - integration-test: test cases to run as the integration test, usually to spot some issues in the scheduler implementation or scheduler-perf itself. +# - performance: test cases to run in the performance test. +# - short: supplemental label for the above two labels (must not used alone), which literally means short execution time test cases. +# +# Specifically, the CIs use labels like the following: +# - `ci-kubernetes-integration-master` (`integration-test`): Test cases are chosen based on a tradeoff between code coverage and overall runtime. +# It basically covers all test cases but with their smallest workload. +# - `pull-kubernetes-integration` (`integration-test`,`short`): Test cases are chosen so that they should take less than total 5 min to complete. +# - `ci-benchmark-scheduler-perf` (`performance`): Long enough test cases are chosen (ideally, longer than 10 seconds) +# to provide meaningful samples for the pod scheduling rate. +# +# Also, `performance`+`short` isn't used in the CIs, but it's used to test the performance test locally. +# (Sometimes, the test cases with `integration-test` are too small to spot issues.) +# +# Combining `performance` and `short` selects suitable workloads for a local +# before/after comparisons with benchstat. + +- name: NodeDeclaredFeaturesEnabled + defaultPodTemplatePath: templates/pod-with-pod-level-resources.yaml + workloadTemplate: + - opcode: createNodes + countParam: $nodes + nodeTemplatePath: templates/node-base.yaml + - opcode: createPods + namespace: init + countParam: $initPods + - opcode: createPods + namespace: test + countParam: $measurePods + collectMetrics: true + workloads: + - name: 5Nodes5DeclaredFeatures + labels: [integration-test, short] + featureGates: + NodeDeclaredFeatures: true + params: + nodes: 5 + initPods: 0 + measurePods: 10 + numNodeDeclaredFeatures: 5 + - name: 5000Nodes20DeclaredFeatures + labels: [performance] + featureGates: + NodeDeclaredFeatures: true + params: + nodes: 5000 + initPods: 5000 + measurePods: 5000 + numNodeDeclaredFeatures: 20 + - name: 5000Nodes40DeclaredFeatures + labels: [performance] + featureGates: + NodeDeclaredFeatures: true + params: + nodes: 5000 + initPods: 5000 + measurePods: 5000 + numNodeDeclaredFeatures: 40 + - name: 5000Nodes100DeclaredFeatures + labels: [performance] + featureGates: + NodeDeclaredFeatures: true + params: + nodes: 5000 + initPods: 5000 + measurePods: 5000 + numNodeDeclaredFeatures: 100 diff --git a/test/integration/scheduler_perf/nodedeclaredfeatures/templates/node-base.yaml b/test/integration/scheduler_perf/nodedeclaredfeatures/templates/node-base.yaml new file mode 100644 index 00000000000..98446ad12bd --- /dev/null +++ b/test/integration/scheduler_perf/nodedeclaredfeatures/templates/node-base.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Node +metadata: + generateName: node- +spec: {} +status: + capacity: + cpu: "128" + memory: "16Gi" + pods: "110" + allocatable: + cpu: "128" + memory: "16Gi" + pods: "110" + phase: Running + # DeclaredFeatures will be added dynamically by the test based on numNodeDeclaredFeatures param in the config. + diff --git a/test/integration/scheduler_perf/nodedeclaredfeatures/templates/pod-with-pod-level-resources.yaml b/test/integration/scheduler_perf/nodedeclaredfeatures/templates/pod-with-pod-level-resources.yaml new file mode 100644 index 00000000000..ad1ca22fc4d --- /dev/null +++ b/test/integration/scheduler_perf/nodedeclaredfeatures/templates/pod-with-pod-level-resources.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod- +spec: + resources: + requests: + cpu: "100m" + memory: "200Mi" + containers: + - name: container-a + image: registry.k8s.io/pause:3.10.1