From e30f9648cc6c60d043941e4c78fec1a1d3619ce3 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Tue, 2 Nov 2021 14:28:06 -0700 Subject: [PATCH] sched: ensure feature gate is honored when instantiating scheduler (#105915) * sched: ensure feature gate is honored when instantiating scheduler * fixup: address comments --- cmd/kube-scheduler/app/options/deprecated.go | 27 ++-- cmd/kube-scheduler/app/options/options.go | 141 +++++++++++------- .../app/options/options_test.go | 27 ++-- cmd/kube-scheduler/app/server.go | 26 ++-- cmd/kube-scheduler/app/server_test.go | 66 ++++++-- cmd/kube-scheduler/app/testing/testserver.go | 14 +- 6 files changed, 187 insertions(+), 114 deletions(-) diff --git a/cmd/kube-scheduler/app/options/deprecated.go b/cmd/kube-scheduler/app/options/deprecated.go index 3176f3c0bfb..279ed422c2f 100644 --- a/cmd/kube-scheduler/app/options/deprecated.go +++ b/cmd/kube-scheduler/app/options/deprecated.go @@ -22,14 +22,17 @@ import ( "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/validation/field" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" + componentbaseconfig "k8s.io/component-base/config" ) // DeprecatedOptions contains deprecated options and their flags. // TODO remove these fields once the deprecated flags are removed. type DeprecatedOptions struct { - // The fields below here are placeholders for flags that can't be directly - // mapped into componentconfig.KubeSchedulerConfiguration. + componentbaseconfig.DebuggingConfiguration + componentbaseconfig.ClientConnectionConfiguration + // Note that only the deprecated options (lock-object-name and lock-object-namespace) are populated here. + componentbaseconfig.LeaderElectionConfiguration + Port int } @@ -47,21 +50,21 @@ func addDummyInsecureFlags(o *DeprecatedOptions, fs *pflag.FlagSet) { } // AddFlags adds flags for the deprecated options. -func (o *DeprecatedOptions) AddFlags(fs *pflag.FlagSet, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) { +func (o *DeprecatedOptions) AddFlags(fs *pflag.FlagSet) { if o == nil { return } addDummyInsecureFlags(o, fs) - fs.BoolVar(&cfg.EnableProfiling, "profiling", cfg.EnableProfiling, "DEPRECATED: enable profiling via web interface host:port/debug/pprof/. This parameter is ignored if a config file is specified in --config.") - fs.BoolVar(&cfg.EnableContentionProfiling, "contention-profiling", cfg.EnableContentionProfiling, "DEPRECATED: enable lock contention profiling, if profiling is enabled. This parameter is ignored if a config file is specified in --config.") - fs.StringVar(&cfg.ClientConnection.Kubeconfig, "kubeconfig", cfg.ClientConnection.Kubeconfig, "DEPRECATED: path to kubeconfig file with authorization and master location information. This parameter is ignored if a config file is specified in --config.") - fs.StringVar(&cfg.ClientConnection.ContentType, "kube-api-content-type", cfg.ClientConnection.ContentType, "DEPRECATED: content type of requests sent to apiserver. This parameter is ignored if a config file is specified in --config.") - fs.Float32Var(&cfg.ClientConnection.QPS, "kube-api-qps", cfg.ClientConnection.QPS, "DEPRECATED: QPS to use while talking with kubernetes apiserver. This parameter is ignored if a config file is specified in --config.") - fs.Int32Var(&cfg.ClientConnection.Burst, "kube-api-burst", cfg.ClientConnection.Burst, "DEPRECATED: burst to use while talking with kubernetes apiserver. This parameter is ignored if a config file is specified in --config.") - fs.StringVar(&cfg.LeaderElection.ResourceNamespace, "lock-object-namespace", cfg.LeaderElection.ResourceNamespace, "DEPRECATED: define the namespace of the lock object. Will be removed in favor of leader-elect-resource-namespace. This parameter is ignored if a config file is specified in --config.") - fs.StringVar(&cfg.LeaderElection.ResourceName, "lock-object-name", cfg.LeaderElection.ResourceName, "DEPRECATED: define the name of the lock object. Will be removed in favor of leader-elect-resource-name. This parameter is ignored if a config file is specified in --config.") + fs.BoolVar(&o.EnableProfiling, "profiling", true, "DEPRECATED: enable profiling via web interface host:port/debug/pprof/. This parameter is ignored if a config file is specified in --config.") + fs.BoolVar(&o.EnableContentionProfiling, "contention-profiling", true, "DEPRECATED: enable lock contention profiling, if profiling is enabled. This parameter is ignored if a config file is specified in --config.") + fs.StringVar(&o.Kubeconfig, "kubeconfig", "", "DEPRECATED: path to kubeconfig file with authorization and master location information. This parameter is ignored if a config file is specified in --config.") + fs.StringVar(&o.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "DEPRECATED: content type of requests sent to apiserver. This parameter is ignored if a config file is specified in --config.") + fs.Float32Var(&o.QPS, "kube-api-qps", 50.0, "DEPRECATED: QPS to use while talking with kubernetes apiserver. This parameter is ignored if a config file is specified in --config.") + fs.Int32Var(&o.Burst, "kube-api-burst", 100, "DEPRECATED: burst to use while talking with kubernetes apiserver. This parameter is ignored if a config file is specified in --config.") + fs.StringVar(&o.ResourceNamespace, "lock-object-namespace", "kube-system", "DEPRECATED: define the namespace of the lock object. Will be removed in favor of leader-elect-resource-namespace. This parameter is ignored if a config file is specified in --config.") + fs.StringVar(&o.ResourceName, "lock-object-name", "kube-scheduler", "DEPRECATED: define the name of the lock object. Will be removed in favor of leader-elect-resource-name. This parameter is ignored if a config file is specified in --config.") } // Validate validates the deprecated scheduler options. diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 4a04c52cb32..d1809bf4f59 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -23,6 +23,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" apiserveroptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -43,7 +44,6 @@ import ( schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" "k8s.io/kubernetes/pkg/scheduler" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - "k8s.io/kubernetes/pkg/scheduler/apis/config/latest" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" netutils "k8s.io/utils/net" ) @@ -51,7 +51,7 @@ import ( // Options has all the params needed to run a Scheduler type Options struct { // The default values. - ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration + ComponentConfig *kubeschedulerconfig.KubeSchedulerConfiguration SecureServing *apiserveroptions.SecureServingOptionsWithLoopback Authentication *apiserveroptions.DelegatingAuthenticationOptions @@ -59,6 +59,7 @@ type Options struct { Metrics *metrics.Options Logs *logs.Options Deprecated *DeprecatedOptions + LeaderElection *componentbaseconfig.LeaderElectionConfiguration // ConfigFile is the location of the scheduler server's configuration file. ConfigFile string @@ -67,17 +68,29 @@ type Options struct { WriteConfigTo string Master string + + // Flags hold the parsed CLI flags. + Flags *cliflag.NamedFlagSets } // NewOptions returns default scheduler app options. -func NewOptions() (*Options, error) { +func NewOptions() *Options { o := &Options{ SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), Deprecated: &DeprecatedOptions{}, - Metrics: metrics.NewOptions(), - Logs: logs.NewOptions(), + LeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: true, + LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, + ResourceLock: "leases", + ResourceName: "kube-scheduler", + ResourceNamespace: "kube-system", + }, + Metrics: metrics.NewOptions(), + Logs: logs.NewOptions(), } o.Authentication.TolerateInClusterLookupFailure = true @@ -89,73 +102,84 @@ func NewOptions() (*Options, error) { o.SecureServing.ServerCert.PairName = "kube-scheduler" o.SecureServing.BindPort = kubeschedulerconfig.DefaultKubeSchedulerPort - return o, nil + o.initFlags() + + return o } -// Complete completes the remaining instantiation of the options obj. -// In particular, it injects the latest internal versioned ComponentConfig. -func (o *Options) Complete(nfs *cliflag.NamedFlagSets) error { - cfg, err := latest.Default() - if err != nil { - return err +// ApplyDeprecated obtains the deprecated CLI args and set them to `o.ComponentConfig` if specified. +func (o *Options) ApplyDeprecated() { + if o.Flags == nil { + return } - // Obtain deprecated CLI args. Set them to cfg if specified in command line. - deprecated := nfs.FlagSet("deprecated") + deprecated := o.Flags.FlagSet("deprecated") if deprecated.Changed("profiling") { - cfg.EnableProfiling = o.ComponentConfig.EnableProfiling + o.ComponentConfig.EnableProfiling = o.Deprecated.EnableProfiling } if deprecated.Changed("contention-profiling") { - cfg.EnableContentionProfiling = o.ComponentConfig.EnableContentionProfiling + o.ComponentConfig.EnableContentionProfiling = o.Deprecated.EnableContentionProfiling } if deprecated.Changed("kubeconfig") { - cfg.ClientConnection.Kubeconfig = o.ComponentConfig.ClientConnection.Kubeconfig + o.ComponentConfig.ClientConnection.Kubeconfig = o.Deprecated.Kubeconfig } if deprecated.Changed("kube-api-content-type") { - cfg.ClientConnection.ContentType = o.ComponentConfig.ClientConnection.ContentType + o.ComponentConfig.ClientConnection.ContentType = o.Deprecated.ContentType } if deprecated.Changed("kube-api-qps") { - cfg.ClientConnection.QPS = o.ComponentConfig.ClientConnection.QPS + o.ComponentConfig.ClientConnection.QPS = o.Deprecated.QPS } if deprecated.Changed("kube-api-burst") { - cfg.ClientConnection.Burst = o.ComponentConfig.ClientConnection.Burst + o.ComponentConfig.ClientConnection.Burst = o.Deprecated.Burst } if deprecated.Changed("lock-object-namespace") { - cfg.LeaderElection.ResourceNamespace = o.ComponentConfig.LeaderElection.ResourceNamespace + o.ComponentConfig.LeaderElection.ResourceNamespace = o.Deprecated.ResourceNamespace } if deprecated.Changed("lock-object-name") { - cfg.LeaderElection.ResourceName = o.ComponentConfig.LeaderElection.ResourceName + o.ComponentConfig.LeaderElection.ResourceName = o.Deprecated.ResourceName } - // Obtain CLI args related with leaderelection. Set them to cfg if specified in command line. - leaderelection := nfs.FlagSet("leader election") - if leaderelection.Changed("leader-elect") { - cfg.LeaderElection.LeaderElect = o.ComponentConfig.LeaderElection.LeaderElect - } - if leaderelection.Changed("leader-elect-lease-duration") { - cfg.LeaderElection.LeaseDuration = o.ComponentConfig.LeaderElection.LeaseDuration - } - if leaderelection.Changed("leader-elect-renew-deadline") { - cfg.LeaderElection.RenewDeadline = o.ComponentConfig.LeaderElection.RenewDeadline - } - if leaderelection.Changed("leader-elect-retry-period") { - cfg.LeaderElection.RetryPeriod = o.ComponentConfig.LeaderElection.RetryPeriod - } - if leaderelection.Changed("leader-elect-resource-lock") { - cfg.LeaderElection.ResourceLock = o.ComponentConfig.LeaderElection.ResourceLock - } - if leaderelection.Changed("leader-elect-resource-name") { - cfg.LeaderElection.ResourceName = o.ComponentConfig.LeaderElection.ResourceName - } - if leaderelection.Changed("leader-elect-resource-namespace") { - cfg.LeaderElection.ResourceNamespace = o.ComponentConfig.LeaderElection.ResourceNamespace - } - - o.ComponentConfig = *cfg - return nil } -// Flags returns flags for a specific scheduler by section name -func (o *Options) Flags() (nfs cliflag.NamedFlagSets) { +// ApplyLeaderElectionTo obtains the CLI args related with leaderelection, and override the values in `cfg`. +// Then the `cfg` object is injected into the `options` object. +func (o *Options) ApplyLeaderElectionTo(cfg *kubeschedulerconfig.KubeSchedulerConfiguration) { + if o.Flags == nil { + return + } + // Obtain CLI args related with leaderelection. Set them to `cfg` if specified in command line. + leaderelection := o.Flags.FlagSet("leader election") + if leaderelection.Changed("leader-elect") { + cfg.LeaderElection.LeaderElect = o.LeaderElection.LeaderElect + } + if leaderelection.Changed("leader-elect-lease-duration") { + cfg.LeaderElection.LeaseDuration = o.LeaderElection.LeaseDuration + } + if leaderelection.Changed("leader-elect-renew-deadline") { + cfg.LeaderElection.RenewDeadline = o.LeaderElection.RenewDeadline + } + if leaderelection.Changed("leader-elect-retry-period") { + cfg.LeaderElection.RetryPeriod = o.LeaderElection.RetryPeriod + } + if leaderelection.Changed("leader-elect-resource-lock") { + cfg.LeaderElection.ResourceLock = o.LeaderElection.ResourceLock + } + if leaderelection.Changed("leader-elect-resource-name") { + cfg.LeaderElection.ResourceName = o.LeaderElection.ResourceName + } + if leaderelection.Changed("leader-elect-resource-namespace") { + cfg.LeaderElection.ResourceNamespace = o.LeaderElection.ResourceNamespace + } + + o.ComponentConfig = cfg +} + +// initFlags initializes flags by section name. +func (o *Options) initFlags() { + if o.Flags != nil { + return + } + + nfs := cliflag.NamedFlagSets{} fs := nfs.FlagSet("misc") fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.") fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.") @@ -164,25 +188,30 @@ func (o *Options) Flags() (nfs cliflag.NamedFlagSets) { o.SecureServing.AddFlags(nfs.FlagSet("secure serving")) o.Authentication.AddFlags(nfs.FlagSet("authentication")) o.Authorization.AddFlags(nfs.FlagSet("authorization")) - o.Deprecated.AddFlags(nfs.FlagSet("deprecated"), &o.ComponentConfig) - - options.BindLeaderElectionFlags(&o.ComponentConfig.LeaderElection, nfs.FlagSet("leader election")) + o.Deprecated.AddFlags(nfs.FlagSet("deprecated")) + options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election")) utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate")) o.Metrics.AddFlags(nfs.FlagSet("metrics")) o.Logs.AddFlags(nfs.FlagSet("logs")) - return nfs + o.Flags = &nfs } // ApplyTo applies the scheduler options to the given scheduler app configuration. func (o *Options) ApplyTo(c *schedulerappconfig.Config) error { if len(o.ConfigFile) == 0 { - c.ComponentConfig = o.ComponentConfig + // If the --config arg is not specified, honor the deprecated as well as leader election CLI args. + o.ApplyDeprecated() + o.ApplyLeaderElectionTo(o.ComponentConfig) + c.ComponentConfig = *o.ComponentConfig } else { cfg, err := loadConfigFromFile(o.ConfigFile) if err != nil { return err } + // If the --config arg is specified, honor the leader election CLI args only. + o.ApplyLeaderElectionTo(cfg) + if err := validation.ValidateKubeSchedulerConfiguration(cfg); err != nil { return err } @@ -210,7 +239,7 @@ func (o *Options) ApplyTo(c *schedulerappconfig.Config) error { func (o *Options) Validate() []error { var errs []error - if err := validation.ValidateKubeSchedulerConfiguration(&o.ComponentConfig); err != nil { + if err := validation.ValidateKubeSchedulerConfiguration(o.ComponentConfig); err != nil { errs = append(errs, err.Errors()...) } errs = append(errs, o.SecureServing.Validate()...) diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index 11c6f905984..b545a4006b9 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -340,9 +340,9 @@ profiles: name: "v1beta3 config file", options: &Options{ ConfigFile: configFile, - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { + ComponentConfig: func() *kubeschedulerconfig.KubeSchedulerConfiguration { cfg := configtesting.V1beta3ToInternalWithDefaults(t, v1beta3.KubeSchedulerConfiguration{}) - return *cfg + return cfg }(), SecureServing: (&apiserveroptions.SecureServingOptions{ ServerCert: apiserveroptions.GeneratableKeyCert{ @@ -412,9 +412,9 @@ profiles: name: "v1beta2 config file", options: &Options{ ConfigFile: v1beta2VersionConfig, - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { + ComponentConfig: func() *kubeschedulerconfig.KubeSchedulerConfiguration { cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{}) - return *cfg + return cfg }(), SecureServing: (&apiserveroptions.SecureServingOptions{ ServerCert: apiserveroptions.GeneratableKeyCert{ @@ -483,12 +483,12 @@ profiles: name: "config file in componentconfig/v1alpha1", options: &Options{ ConfigFile: oldConfigFile, - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { + ComponentConfig: func() *kubeschedulerconfig.KubeSchedulerConfiguration { cfg, err := latest.Default() if err != nil { t.Fatal(err) } - return *cfg + return cfg }(), Logs: logs.NewOptions(), }, @@ -513,10 +513,10 @@ profiles: { name: "kubeconfig flag", options: &Options{ - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { + ComponentConfig: func() *kubeschedulerconfig.KubeSchedulerConfiguration { cfg, _ := latest.Default() cfg.ClientConnection.Kubeconfig = flagKubeconfig - return *cfg + return cfg }(), SecureServing: (&apiserveroptions.SecureServingOptions{ ServerCert: apiserveroptions.GeneratableKeyCert{ @@ -584,10 +584,10 @@ profiles: { name: "overridden master", options: &Options{ - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { + ComponentConfig: func() *kubeschedulerconfig.KubeSchedulerConfiguration { cfg, _ := latest.Default() cfg.ClientConnection.Kubeconfig = flagKubeconfig - return *cfg + return cfg }(), Master: insecureserver.URL, SecureServing: (&apiserveroptions.SecureServingOptions{ @@ -1149,6 +1149,13 @@ profiles: for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { + if tc.options.ComponentConfig == nil { + if cfg, err := latest.Default(); err != nil { + t.Fatal(err) + } else { + tc.options.ComponentConfig = cfg + } + } // create the config config, err := tc.options.Config() diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index aa935a0b6f9..7e46a7dc1e7 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -53,6 +53,7 @@ import ( "k8s.io/kubernetes/cmd/kube-scheduler/app/options" "k8s.io/kubernetes/pkg/scheduler" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/apis/config/latest" "k8s.io/kubernetes/pkg/scheduler/framework/runtime" "k8s.io/kubernetes/pkg/scheduler/metrics/resources" "k8s.io/kubernetes/pkg/scheduler/profile" @@ -63,13 +64,8 @@ type Option func(runtime.Registry) error // NewSchedulerCommand creates a *cobra.Command object with default parameters and registryOptions func NewSchedulerCommand(registryOptions ...Option) *cobra.Command { - opts, err := options.NewOptions() - if err != nil { - klog.ErrorS(err, "Unable to initialize command options") - os.Exit(1) - } + opts := options.NewOptions() - namedFlagSets := opts.Flags() cmd := &cobra.Command{ Use: "kube-scheduler", Long: `The Kubernetes scheduler is a control plane process which assigns @@ -81,9 +77,6 @@ kube-scheduler is the reference implementation. See [scheduling](https://kubernetes.io/docs/concepts/scheduling-eviction/) for more information about scheduling and the kube-scheduler component.`, RunE: func(cmd *cobra.Command, args []string) error { - if err := opts.Complete(&namedFlagSets); err != nil { - return err - } return runCommand(cmd, opts, registryOptions...) }, Args: func(cmd *cobra.Command, args []string) error { @@ -96,15 +89,16 @@ for more information about scheduling and the kube-scheduler component.`, }, } + nfs := opts.Flags + verflag.AddFlags(nfs.FlagSet("global")) + globalflag.AddGlobalFlags(nfs.FlagSet("global"), cmd.Name()) fs := cmd.Flags() - verflag.AddFlags(namedFlagSets.FlagSet("global")) - globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) - for _, f := range namedFlagSets.FlagSets { + for _, f := range nfs.FlagSets { fs.AddFlagSet(f) } cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) - cliflag.SetUsageAndHelpFunc(cmd, namedFlagSets, cols) + cliflag.SetUsageAndHelpFunc(cmd, *nfs, cols) cmd.MarkFlagFilename("config", "yaml", "yml", "json") @@ -284,6 +278,12 @@ func WithPlugin(name string, factory runtime.PluginFactory) Option { // Setup creates a completed config and a scheduler based on the command args and options func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions ...Option) (*schedulerserverconfig.CompletedConfig, *scheduler.Scheduler, error) { + if cfg, err := latest.Default(); err != nil { + return nil, nil, err + } else { + opts.ComponentConfig = cfg + } + if errs := opts.Validate(); len(errs) > 0 { return nil, nil, utilerrors.NewAggregate(errs) } diff --git a/cmd/kube-scheduler/app/server_test.go b/cmd/kube-scheduler/app/server_test.go index 3bbc590621b..f5e8c061ff6 100644 --- a/cmd/kube-scheduler/app/server_test.go +++ b/cmd/kube-scheduler/app/server_test.go @@ -29,9 +29,15 @@ import ( "github.com/google/go-cmp/cmp" "github.com/spf13/pflag" + "k8s.io/apiserver/pkg/util/feature" + componentbaseconfig "k8s.io/component-base/config" + "k8s.io/component-base/featuregate" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/cmd/kube-scheduler/app/options" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" ) func TestSetup(t *testing.T) { @@ -140,10 +146,45 @@ profiles: } testcases := []struct { - name string - flags []string - wantPlugins map[string]*config.Plugins + name string + flags []string + restoreFeatures map[featuregate.Feature]bool + wantPlugins map[string]*config.Plugins + wantLeaderElection *componentbaseconfig.LeaderElectionConfiguration }{ + { + name: "default config with an alpha feature enabled and an beta feature disabled", + flags: []string{ + "--kubeconfig", configKubeconfig, + "--feature-gates=VolumeCapacityPriority=true,DefaultPodTopologySpread=false", + }, + wantPlugins: map[string]*config.Plugins{ + "default-scheduler": func() *config.Plugins { + plugins := &config.Plugins{ + QueueSort: defaults.PluginsV1beta3.QueueSort, + PreFilter: defaults.PluginsV1beta3.PreFilter, + Filter: defaults.PluginsV1beta3.Filter, + PostFilter: defaults.PluginsV1beta3.PostFilter, + PreScore: defaults.PluginsV1beta3.PreScore, + Score: defaults.PluginsV1beta3.Score, + Bind: defaults.PluginsV1beta3.Bind, + PreBind: defaults.PluginsV1beta3.PreBind, + Reserve: defaults.PluginsV1beta3.Reserve, + } + plugins.PreScore.Enabled = append(plugins.PreScore.Enabled, config.Plugin{Name: names.SelectorSpread, Weight: 0}) + plugins.Score.Enabled = append( + plugins.Score.Enabled, + config.Plugin{Name: names.VolumeBinding, Weight: 1}, + config.Plugin{Name: names.SelectorSpread, Weight: 1}, + ) + return plugins + }(), + }, + restoreFeatures: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: false, + features.DefaultPodTopologySpread: true, + }, + }, { name: "default config", flags: []string{ @@ -206,13 +247,18 @@ profiles: for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - fs := pflag.NewFlagSet("test", pflag.PanicOnError) - opts, err := options.NewOptions() - if err != nil { - t.Fatal(err) + for k, v := range tc.restoreFeatures { + defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)() } - nfs := opts.Flags() + fs := pflag.NewFlagSet("test", pflag.PanicOnError) + opts := options.NewOptions() + + // use listeners instead of static ports so parallel test runs don't conflict + opts.SecureServing.Listener = makeListener(t) + defer opts.SecureServing.Listener.Close() + + nfs := opts.Flags for _, f := range nfs.FlagSets { fs.AddFlagSet(f) } @@ -220,10 +266,6 @@ profiles: t.Fatal(err) } - if err := opts.Complete(&nfs); err != nil { - t.Fatal(err) - } - // use listeners instead of static ports so parallel test runs don't conflict opts.SecureServing.Listener = makeListener(t) defer opts.SecureServing.Listener.Close() diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go index b3c0a309c38..8cc8bdd6c79 100644 --- a/cmd/kube-scheduler/app/testing/testserver.go +++ b/cmd/kube-scheduler/app/testing/testserver.go @@ -82,21 +82,13 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err fs := pflag.NewFlagSet("test", pflag.PanicOnError) - opts, err := options.NewOptions() - if err != nil { - return TestServer{}, err - } - - namedFlagSets := opts.Flags() - for _, f := range namedFlagSets.FlagSets { + opts := options.NewOptions() + nfs := opts.Flags + for _, f := range nfs.FlagSets { fs.AddFlagSet(f) } fs.Parse(customFlags) - if err := opts.Complete(&namedFlagSets); err != nil { - return TestServer{}, err - } - if opts.SecureServing.BindPort != 0 { opts.SecureServing.Listener, opts.SecureServing.BindPort, err = createListenerOnFreePort() if err != nil {