From d50c0aeb5f134e0e2468aa70662d24f8b7280822 Mon Sep 17 00:00:00 2001 From: Dave Chen Date: Tue, 2 Mar 2021 18:52:30 +0800 Subject: [PATCH] Enable scheduler_perf to support scheduler config file Signed-off-by: Dave Chen --- cmd/kube-scheduler/app/options/configfile.go | 2 +- .../scheduler_perf_legacy_test.go | 2 +- .../scheduler_perf/scheduler_perf_test.go | 34 ++++++++++++++++++- .../scheduler_perf/scheduler_test.go | 2 +- test/integration/scheduler_perf/util.go | 30 ++++++++++++++-- test/integration/util/util.go | 12 +++++-- 6 files changed, 74 insertions(+), 8 deletions(-) diff --git a/cmd/kube-scheduler/app/options/configfile.go b/cmd/kube-scheduler/app/options/configfile.go index 5cd6bf72412..578a34d97a3 100644 --- a/cmd/kube-scheduler/app/options/configfile.go +++ b/cmd/kube-scheduler/app/options/configfile.go @@ -19,10 +19,10 @@ package options import ( "fmt" "io/ioutil" - "k8s.io/klog/v2" "os" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" kubeschedulerconfigv1beta1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta1" diff --git a/test/integration/scheduler_perf/scheduler_perf_legacy_test.go b/test/integration/scheduler_perf/scheduler_perf_legacy_test.go index 34c6786b451..42267238c35 100644 --- a/test/integration/scheduler_perf/scheduler_perf_legacy_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_legacy_test.go @@ -442,7 +442,7 @@ func benchmarkScheduling(numExistingPods, minPods int, //lint:ignore SA3001 Set a minimum for b.N to get more meaningful results b.N = minPods } - finalFunc, podInformer, clientset, _ := mustSetupScheduler() + finalFunc, podInformer, clientset, _ := mustSetupScheduler(nil) defer finalFunc() nodePreparer := framework.NewIntegrationTestNodePreparer( diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index 8bd4b41d717..5d4cf43ab78 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -43,6 +43,9 @@ import ( "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" + "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" "k8s.io/kubernetes/test/integration/framework" testutils "k8s.io/kubernetes/test/utils" "sigs.k8s.io/yaml" @@ -91,6 +94,8 @@ type testCase struct { WorkloadTemplate []op // List of workloads to run under this testCase. Workloads []*workload + // SchedulerConfigFile is the path of scheduler configuration + SchedulerConfigFile string // TODO(#93792): reduce config toil by having a default pod and node spec per // testCase? CreatePods and CreateNodes ops will inherit these unless // manually overridden. @@ -372,11 +377,38 @@ func BenchmarkPerfScheduling(b *testing.B) { } } +func loadSchedulerConfig(file string) (*config.KubeSchedulerConfiguration, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + // The UniversalDecoder runs defaulting and returns the internal type by default. + obj, gvk, err := scheme.Codecs.UniversalDecoder().Decode(data, nil, nil) + if err != nil { + return nil, err + } + if cfgObj, ok := obj.(*config.KubeSchedulerConfiguration); ok { + return cfgObj, nil + } + return nil, fmt.Errorf("couldn't decode as KubeSchedulerConfiguration, got %s: ", gvk) +} + func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem { // 30 minutes should be plenty enough even for the 5000-node tests. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - finalFunc, podInformer, client, dynClient := mustSetupScheduler() + var cfg *config.KubeSchedulerConfiguration + var err error + if len(tc.SchedulerConfigFile) != 0 { + cfg, err = loadSchedulerConfig(tc.SchedulerConfigFile) + if err != nil { + b.Fatalf("error loading scheduler config file: %v", err) + } + if err = validation.ValidateKubeSchedulerConfiguration(cfg).ToAggregate(); err != nil { + b.Fatalf("validate scheduler config file failed: %v", err) + } + } + finalFunc, podInformer, client, dynClient := mustSetupScheduler(cfg) b.Cleanup(finalFunc) var mu sync.Mutex diff --git a/test/integration/scheduler_perf/scheduler_test.go b/test/integration/scheduler_perf/scheduler_test.go index f75c2ce81ce..93f2907b428 100644 --- a/test/integration/scheduler_perf/scheduler_test.go +++ b/test/integration/scheduler_perf/scheduler_test.go @@ -116,7 +116,7 @@ type testConfig struct { // getBaseConfig returns baseConfig after initializing number of nodes and pods. func getBaseConfig(nodes int, pods int) *testConfig { - destroyFunc, podInformer, clientset, _ := mustSetupScheduler() + destroyFunc, podInformer, clientset, _ := mustSetupScheduler(nil) return &testConfig{ clientset: clientset, destroyFunc: destroyFunc, diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 7fdc4e813b1..9ba21eae332 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -40,6 +40,9 @@ import ( "k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/testutil" "k8s.io/klog/v2" + "k8s.io/kube-scheduler/config/v1beta1" + "k8s.io/kubernetes/pkg/scheduler/apis/config" + kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" "k8s.io/kubernetes/test/integration/util" testutils "k8s.io/kubernetes/test/utils" ) @@ -53,6 +56,16 @@ const ( var dataItemsDir = flag.String("data-items-dir", "", "destination directory for storing generated data items for perf dashboard") +func newDefaultComponentConfig() (*config.KubeSchedulerConfiguration, error) { + gvk := v1beta1.SchemeGroupVersion.WithKind("KubeSchedulerConfiguration") + cfg := config.KubeSchedulerConfiguration{} + _, _, err := kubeschedulerscheme.Codecs.UniversalDecoder().Decode(nil, &gvk, &cfg) + if err != nil { + return nil, err + } + return &cfg, nil +} + // mustSetupScheduler starts the following components: // - k8s api server (a.k.a. master) // - scheduler @@ -60,9 +73,12 @@ var dataItemsDir = flag.String("data-items-dir", "", "destination directory for // remove resources after finished. // Notes on rate limiter: // - client rate limit is set to 5000. -func mustSetupScheduler() (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface, dynamic.Interface) { +func mustSetupScheduler(config *config.KubeSchedulerConfiguration) (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface, dynamic.Interface) { apiURL, apiShutdown := util.StartApiserver() + var err error + // TODO: client connection configuration, such as QPS or Burst is configurable in theory, this could be derived from the `config`, need to + // support this when there is any testcase that depends on such configuration. cfg := &restclient.Config{ Host: apiURL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}, @@ -70,10 +86,20 @@ func mustSetupScheduler() (util.ShutdownFunc, coreinformers.PodInformer, clients Burst: 5000, } + // use default component config if config here is nil + if config == nil { + config, err = newDefaultComponentConfig() + if err != nil { + klog.Fatalf("Error creating default component config: %v", err) + } + } + client := clientset.NewForConfigOrDie(cfg) dynClient := dynamic.NewForConfigOrDie(cfg) - _, podInformer, schedulerShutdown := util.StartScheduler(client) + // Not all config options will be effective but only those mostly related with scheduler performance will + // be applied to start a scheduler, most of them are defined in `scheduler.schedulerOptions`. + _, podInformer, schedulerShutdown := util.StartScheduler(client, config) fakePVControllerShutdown := util.StartFakePVController(client) shutdownFunc := func() { diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 4fe76aee5ef..7967cb29e97 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -42,6 +42,7 @@ import ( "k8s.io/klog/v2" pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/scheduler" + kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" schedulerapiv1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1" @@ -74,7 +75,7 @@ func StartApiserver() (string, ShutdownFunc) { // StartScheduler configures and starts a scheduler given a handle to the clientSet interface // and event broadcaster. It returns the running scheduler, podInformer and the shutdown function to stop it. -func StartScheduler(clientSet clientset.Interface) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) { +func StartScheduler(clientSet clientset.Interface, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) { ctx, cancel := context.WithCancel(context.Background()) informerFactory := scheduler.NewInformerFactory(clientSet, 0) @@ -87,7 +88,14 @@ func StartScheduler(clientSet clientset.Interface) (*scheduler.Scheduler, corein clientSet, informerFactory, profile.NewRecorderFactory(evtBroadcaster), - ctx.Done()) + ctx.Done(), + scheduler.WithProfiles(cfg.Profiles...), + scheduler.WithAlgorithmSource(cfg.AlgorithmSource), + scheduler.WithPercentageOfNodesToScore(cfg.PercentageOfNodesToScore), + scheduler.WithPodMaxBackoffSeconds(cfg.PodMaxBackoffSeconds), + scheduler.WithPodInitialBackoffSeconds(cfg.PodInitialBackoffSeconds), + scheduler.WithExtenders(cfg.Extenders...), + scheduler.WithParallelism(cfg.Parallelism)) if err != nil { klog.Fatalf("Error creating scheduler: %v", err) }