Add Node Declared Features scheduler performance tests

test/integration/scheduler_perf/misc/performance-config.yaml

@@ -100,7 +100,14 @@
       initNodes: 5000
       initPods: 5000
       measurePods: 50000
-
+  - name: 5000Nodes_50000Pods_NodeDeclaredFeaturesDisabled
+    labels: [performance]
+    featureGates:
+      NodeDeclaredFeatures: false
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 50000
 # This test case simulates the scheduling of daemonset.
 # https://github.com/kubernetes/kubernetes/issues/124709
 - name: SchedulingDaemonset

test/integration/scheduler_perf/nodedeclaredfeatures/node_declared_features_test.go

@@ -0,0 +1,187 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodedeclaredfeatures
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/util/wait"
+	_ "k8s.io/component-base/logs/json/register"
+	ndf "k8s.io/component-helpers/nodedeclaredfeatures"
+	ndffeatures "k8s.io/component-helpers/nodedeclaredfeatures/features"
+	"k8s.io/kubernetes/pkg/features"
+	perf "k8s.io/kubernetes/test/integration/scheduler_perf"
+	"k8s.io/kubernetes/test/utils/ktesting"
+)
+
+// This test benchmarks the NodeDeclaredFeatures features performance impact.
+//
+// It mocks a variable number of node features registered (via 'numNodeDeclaredFeatures' param) but the
+// inference function of all the registered features look for pod level resouces ('spec.resources').
+// The tests creates a pod with pod level resources to ensure that all features are inferred
+// during scheduling.
+
+func TestMain(m *testing.M) {
+	if err := perf.InitTests(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+
+	m.Run()
+}
+
+// mockFeature is a mock implementation of the Feature interface for testing.
+type mockFeature struct {
+	name       string
+	maxVersion *version.Version
+}
+
+func (f *mockFeature) Name() string {
+	return f.name
+}
+
+func (f *mockFeature) Discover(cfg *ndf.NodeConfiguration) bool {
+	// This function is called by kubelet to discover features on the node and report them in the node status.
+	// In the test context, feature discovery is bypassed and we update node status directly after
+	// the node is created (see updateNodesWithDeclaredFeatures()), so this function is a no-op.
+	return true
+}
+
+func (f *mockFeature) InferForScheduling(podInfo *ndf.PodInfo) bool {
+	// Check if the pod is using pod level resources.
+	return podInfo.Spec != nil && podInfo.Spec.Resources != nil
+}
+
+func (f *mockFeature) InferForUpdate(oldPodInfo, newPodInfo *ndf.PodInfo) bool {
+	return false
+}
+
+func (f *mockFeature) Requirements() *ndf.FeatureRequirements {
+	return nil
+}
+
+func (f *mockFeature) MaxVersion() *version.Version {
+	return f.maxVersion
+}
+
+func createMockFeature(name string, maxVersionStr string) ndf.Feature {
+	var v *version.Version
+	if maxVersionStr != "" {
+		v = version.MustParseSemantic(maxVersionStr)
+	}
+	return &mockFeature{
+		name:       name,
+		maxVersion: v,
+	}
+}
+
+func setupFeatures(numNodeDeclaredFeatures int) func() {
+	nodeFeatures := make([]ndf.Feature, 0, numNodeDeclaredFeatures)
+	for i := 1; i <= numNodeDeclaredFeatures; i++ {
+		featureName := fmt.Sprintf("Feature%d", i)
+		nodeFeatures = append(nodeFeatures, createMockFeature(featureName, ""))
+	}
+	originalAllFeatures := ndffeatures.AllFeatures
+	ndffeatures.AllFeatures = nodeFeatures
+	return func() {
+		ndffeatures.AllFeatures = originalAllFeatures
+	}
+}
+
+func preInitNodeDeclaredFeatures(t ktesting.TContext, w *perf.Workload) (func(), error) {
+	if !w.FeatureGates[features.NodeDeclaredFeatures] {
+		t.Logf("Skipping NodeDeclaredFeatures pre-init as the feature gate is disabled")
+		return func() {}, nil
+	}
+
+	numNodeDeclaredFeatures, err := w.GetParam("numNodeDeclaredFeatures")
+	if err != nil {
+		t.Logf("numNodeDeclaredFeatures param not specified in workload config")
+		numNodeDeclaredFeatures = 0
+	}
+
+	t.Logf("PreInit: Setting up %d mock features for %s", numNodeDeclaredFeatures, w.Name)
+	// Setup mock features to be registered in the NDF library.
+	cleanupMockFeatures := setupFeatures(numNodeDeclaredFeatures)
+
+	return cleanupMockFeatures, nil
+}
+
+func updateNodesWithDeclaredFeatures(tCtx ktesting.TContext, w *perf.Workload, nodes *v1.NodeList) error {
+	if !w.FeatureGates[features.NodeDeclaredFeatures] {
+		return nil
+	}
+
+	numNodeDeclaredFeatures, err := w.GetParam("numNodeDeclaredFeatures")
+	if err != nil {
+		tCtx.Logf("numNodeDeclaredFeatures param not specified in workload config")
+		numNodeDeclaredFeatures = 0
+	}
+
+	var featureNames []string
+	for i := 1; i <= numNodeDeclaredFeatures; i++ {
+		featureNames = append(featureNames, fmt.Sprintf("Feature%d", i))
+	}
+	sort.Strings(featureNames)
+
+	for _, node := range nodes.Items {
+		nodeToUpdate := node.DeepCopy()
+		nodeToUpdate.Status.DeclaredFeatures = featureNames
+		_, err := tCtx.Client().CoreV1().Nodes().UpdateStatus(tCtx.Context, nodeToUpdate, metav1.UpdateOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to update node %s status: %w", node.Name, err)
+		}
+		tCtx.Logf("Updated node %s status with %d features", node.Name, len(featureNames))
+	}
+
+	err = wait.PollUntilContextTimeout(tCtx.Context, 100*time.Millisecond, 60*time.Second, true, func(ctx context.Context) (bool, error) {
+		for _, n := range nodes.Items {
+			updatedNode, err := tCtx.Client().CoreV1().Nodes().Get(ctx, n.Name, metav1.GetOptions{})
+			if err != nil {
+				return false, err
+			}
+
+			if !reflect.DeepEqual(updatedNode.Status.DeclaredFeatures, featureNames) {
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("timeout waiting for all nodes to reflect status update: %w", err)
+	}
+
+	return nil
+}
+
+func TestSchedulerPerf(t *testing.T) {
+	perf.RunIntegrationPerfScheduling(t, "performance-config.yaml", perf.WithPreRunFn(preInitNodeDeclaredFeatures), perf.WithNodeUpdateFn(updateNodesWithDeclaredFeatures))
+}
+
+func BenchmarkPerfScheduling(b *testing.B) {
+	perf.RunBenchmarkPerfScheduling(b, "performance-config.yaml", "nodedeclaredfeatures", nil, perf.WithPreRunFn(preInitNodeDeclaredFeatures), perf.WithNodeUpdateFn(updateNodesWithDeclaredFeatures))
+}

test/integration/scheduler_perf/nodedeclaredfeatures/performance-config.yaml

@@ -0,0 +1,69 @@
+# The following labels are used in this file. (listed in ascending order of the number of covered test cases)
+#
+# - integration-test: test cases to run as the integration test, usually to spot some issues in the scheduler implementation or scheduler-perf itself.
+# - performance: test cases to run in the performance test.
+# - short: supplemental label for the above two labels (must not used alone), which literally means short execution time test cases.
+#
+# Specifically, the CIs use labels like the following:
+# - `ci-kubernetes-integration-master` (`integration-test`): Test cases are chosen based on a tradeoff between code coverage and overall runtime. 
+# It basically covers all test cases but with their smallest workload. 
+# - `pull-kubernetes-integration` (`integration-test`,`short`): Test cases are chosen so that they should take less than total 5 min to complete.
+# - `ci-benchmark-scheduler-perf` (`performance`): Long enough test cases are chosen (ideally, longer than 10 seconds) 
+# to provide meaningful samples for the pod scheduling rate.
+#
+# Also, `performance`+`short` isn't used in the CIs, but it's used to test the performance test locally.
+# (Sometimes, the test cases with `integration-test` are too small to spot issues.)
+#
+# Combining `performance` and `short` selects suitable workloads for a local
+# before/after comparisons with benchstat.
+
+- name: NodeDeclaredFeaturesEnabled
+  defaultPodTemplatePath: templates/pod-with-pod-level-resources.yaml
+  workloadTemplate:
+    - opcode: createNodes
+      countParam: $nodes
+      nodeTemplatePath: templates/node-base.yaml
+    - opcode: createPods
+      namespace: init
+      countParam: $initPods
+    - opcode: createPods
+      namespace: test
+      countParam: $measurePods
+      collectMetrics: true
+  workloads:
+    - name: 5Nodes5DeclaredFeatures
+      labels: [integration-test, short]
+      featureGates:
+        NodeDeclaredFeatures: true
+      params:
+        nodes: 5
+        initPods: 0
+        measurePods: 10
+        numNodeDeclaredFeatures: 5
+    - name: 5000Nodes20DeclaredFeatures
+      labels: [performance]
+      featureGates:
+        NodeDeclaredFeatures: true
+      params:
+        nodes: 5000
+        initPods: 5000
+        measurePods: 5000
+        numNodeDeclaredFeatures: 20
+    - name: 5000Nodes40DeclaredFeatures
+      labels: [performance]
+      featureGates:
+        NodeDeclaredFeatures: true
+      params:
+        nodes: 5000
+        initPods: 5000
+        measurePods: 5000
+        numNodeDeclaredFeatures: 40
+    - name: 5000Nodes100DeclaredFeatures
+      labels: [performance]
+      featureGates:
+        NodeDeclaredFeatures: true
+      params:
+        nodes: 5000
+        initPods: 5000
+        measurePods: 5000
+        numNodeDeclaredFeatures: 100

test/integration/scheduler_perf/nodedeclaredfeatures/templates/node-base.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Node
+metadata:
+  generateName: node-
+spec: {}
+status:
+  capacity:
+    cpu: "128"
+    memory: "16Gi"
+    pods: "110"
+  allocatable:
+    cpu: "128"
+    memory: "16Gi"
+    pods: "110"
+  phase: Running
+  # DeclaredFeatures will be added dynamically by the test based on numNodeDeclaredFeatures param in the config.
+

test/integration/scheduler_perf/nodedeclaredfeatures/templates/pod-with-pod-level-resources.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-
+spec:
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "200Mi"
+  containers:
+  - name: container-a
+    image: registry.k8s.io/pause:3.10.1