Enable scheduler_perf to support scheduler config file

Signed-off-by: Dave Chen <dave.chen@arm.com>

cmd/kube-scheduler/app/options/configfile.go

@@ -19,10 +19,10 @@ package options
 import (
 	"fmt"
 	"io/ioutil"
-	"k8s.io/klog/v2"
 	"os"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
 	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 	kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
 	kubeschedulerconfigv1beta1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta1"

test/integration/scheduler_perf/scheduler_perf_legacy_test.go

@@ -442,7 +442,7 @@ func benchmarkScheduling(numExistingPods, minPods int,
 		//lint:ignore SA3001 Set a minimum for b.N to get more meaningful results
 		b.N = minPods
 	}
-	finalFunc, podInformer, clientset, _ := mustSetupScheduler()
+	finalFunc, podInformer, clientset, _ := mustSetupScheduler(nil)
 	defer finalFunc()
 
 	nodePreparer := framework.NewIntegrationTestNodePreparer(

test/integration/scheduler_perf/scheduler_perf_test.go

@@ -43,6 +43,9 @@ import (
 	"k8s.io/component-base/featuregate"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/scheduler/apis/config"
+	"k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
+	"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
 	"k8s.io/kubernetes/test/integration/framework"
 	testutils "k8s.io/kubernetes/test/utils"
 	"sigs.k8s.io/yaml"
@@ -91,6 +94,8 @@ type testCase struct {
 	WorkloadTemplate []op
 	// List of workloads to run under this testCase.
 	Workloads []*workload
+	// SchedulerConfigFile is the path of scheduler configuration
+	SchedulerConfigFile string
 	// TODO(#93792): reduce config toil by having a default pod and node spec per
 	// testCase? CreatePods and CreateNodes ops will inherit these unless
 	// manually overridden.
@@ -372,11 +377,38 @@ func BenchmarkPerfScheduling(b *testing.B) {
 	}
 }
 
+func loadSchedulerConfig(file string) (*config.KubeSchedulerConfiguration, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	// The UniversalDecoder runs defaulting and returns the internal type by default.
+	obj, gvk, err := scheme.Codecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	if cfgObj, ok := obj.(*config.KubeSchedulerConfiguration); ok {
+		return cfgObj, nil
+	}
+	return nil, fmt.Errorf("couldn't decode as KubeSchedulerConfiguration, got %s: ", gvk)
+}
+
 func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem {
 	// 30 minutes should be plenty enough even for the 5000-node tests.
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
-	finalFunc, podInformer, client, dynClient := mustSetupScheduler()
+	var cfg *config.KubeSchedulerConfiguration
+	var err error
+	if len(tc.SchedulerConfigFile) != 0 {
+		cfg, err = loadSchedulerConfig(tc.SchedulerConfigFile)
+		if err != nil {
+			b.Fatalf("error loading scheduler config file: %v", err)
+		}
+		if err = validation.ValidateKubeSchedulerConfiguration(cfg).ToAggregate(); err != nil {
+			b.Fatalf("validate scheduler config file failed: %v", err)
+		}
+	}
+	finalFunc, podInformer, client, dynClient := mustSetupScheduler(cfg)
 	b.Cleanup(finalFunc)
 
 	var mu sync.Mutex

test/integration/scheduler_perf/scheduler_test.go

@@ -116,7 +116,7 @@ type testConfig struct {
 
 // getBaseConfig returns baseConfig after initializing number of nodes and pods.
 func getBaseConfig(nodes int, pods int) *testConfig {
-	destroyFunc, podInformer, clientset, _ := mustSetupScheduler()
+	destroyFunc, podInformer, clientset, _ := mustSetupScheduler(nil)
 	return &testConfig{
 		clientset:   clientset,
 		destroyFunc: destroyFunc,

test/integration/scheduler_perf/util.go

@@ -40,6 +40,9 @@ import (
 	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/component-base/metrics/testutil"
 	"k8s.io/klog/v2"
+	"k8s.io/kube-scheduler/config/v1beta1"
+	"k8s.io/kubernetes/pkg/scheduler/apis/config"
+	kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
 	"k8s.io/kubernetes/test/integration/util"
 	testutils "k8s.io/kubernetes/test/utils"
 )
@@ -53,6 +56,16 @@ const (
 
 var dataItemsDir = flag.String("data-items-dir", "", "destination directory for storing generated data items for perf dashboard")
 
+func newDefaultComponentConfig() (*config.KubeSchedulerConfiguration, error) {
+	gvk := v1beta1.SchemeGroupVersion.WithKind("KubeSchedulerConfiguration")
+	cfg := config.KubeSchedulerConfiguration{}
+	_, _, err := kubeschedulerscheme.Codecs.UniversalDecoder().Decode(nil, &gvk, &cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
+
 // mustSetupScheduler starts the following components:
 // - k8s api server (a.k.a. master)
 // - scheduler
@@ -60,9 +73,12 @@ var dataItemsDir = flag.String("data-items-dir", "", "destination directory for
 // remove resources after finished.
 // Notes on rate limiter:
 //   - client rate limit is set to 5000.
-func mustSetupScheduler() (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface, dynamic.Interface) {
+func mustSetupScheduler(config *config.KubeSchedulerConfiguration) (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface, dynamic.Interface) {
 	apiURL, apiShutdown := util.StartApiserver()
+	var err error
 
+	// TODO: client connection configuration, such as QPS or Burst is configurable in theory, this could be derived from the `config`, need to
+	// support this when there is any testcase that depends on such configuration.
 	cfg := &restclient.Config{
 		Host:          apiURL,
 		ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}},
@@ -70,10 +86,20 @@ func mustSetupScheduler() (util.ShutdownFunc, coreinformers.PodInformer, clients
 		Burst:         5000,
 	}
 
+	// use default component config if config here is nil
+	if config == nil {
+		config, err = newDefaultComponentConfig()
+		if err != nil {
+			klog.Fatalf("Error creating default component config: %v", err)
+		}
+	}
+
 	client := clientset.NewForConfigOrDie(cfg)
 	dynClient := dynamic.NewForConfigOrDie(cfg)
 
-	_, podInformer, schedulerShutdown := util.StartScheduler(client)
+	// Not all config options will be effective but only those mostly related with scheduler performance will
+	// be applied to start a scheduler, most of them are defined in `scheduler.schedulerOptions`.
+	_, podInformer, schedulerShutdown := util.StartScheduler(client, config)
 	fakePVControllerShutdown := util.StartFakePVController(client)
 
 	shutdownFunc := func() {

test/integration/util/util.go

@@ -42,6 +42,7 @@ import (
 	"k8s.io/klog/v2"
 	pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util"
 	"k8s.io/kubernetes/pkg/scheduler"
+	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
 	schedulerapiv1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1"
@@ -74,7 +75,7 @@ func StartApiserver() (string, ShutdownFunc) {
 
 // StartScheduler configures and starts a scheduler given a handle to the clientSet interface
 // and event broadcaster. It returns the running scheduler, podInformer and the shutdown function to stop it.
-func StartScheduler(clientSet clientset.Interface) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) {
+func StartScheduler(clientSet clientset.Interface, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	informerFactory := scheduler.NewInformerFactory(clientSet, 0)
@@ -87,7 +88,14 @@ func StartScheduler(clientSet clientset.Interface) (*scheduler.Scheduler, corein
 		clientSet,
 		informerFactory,
 		profile.NewRecorderFactory(evtBroadcaster),
-		ctx.Done())
+		ctx.Done(),
+		scheduler.WithProfiles(cfg.Profiles...),
+		scheduler.WithAlgorithmSource(cfg.AlgorithmSource),
+		scheduler.WithPercentageOfNodesToScore(cfg.PercentageOfNodesToScore),
+		scheduler.WithPodMaxBackoffSeconds(cfg.PodMaxBackoffSeconds),
+		scheduler.WithPodInitialBackoffSeconds(cfg.PodInitialBackoffSeconds),
+		scheduler.WithExtenders(cfg.Extenders...),
+		scheduler.WithParallelism(cfg.Parallelism))
 	if err != nil {
 		klog.Fatalf("Error creating scheduler: %v", err)
 	}