diff --git a/cmd/kube-scheduler/app/BUILD b/cmd/kube-scheduler/app/BUILD index 3e3d92597d8..db61d56183d 100644 --- a/cmd/kube-scheduler/app/BUILD +++ b/cmd/kube-scheduler/app/BUILD @@ -61,6 +61,7 @@ filegroup( ":package-srcs", "//cmd/kube-scheduler/app/config:all-srcs", "//cmd/kube-scheduler/app/options:all-srcs", + "//cmd/kube-scheduler/app/testing:all-srcs", ], tags = ["automanaged"], ) diff --git a/cmd/kube-scheduler/app/config/BUILD b/cmd/kube-scheduler/app/config/BUILD index 6545f5ff84a..877ceef1e05 100644 --- a/cmd/kube-scheduler/app/config/BUILD +++ b/cmd/kube-scheduler/app/config/BUILD @@ -12,6 +12,7 @@ go_library( "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", ], diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 930759d28c0..2ad5ae9bf34 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -22,6 +22,7 @@ import ( coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/record" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -32,6 +33,9 @@ type Config struct { // config is the scheduler server's configuration object. ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration + // LoopbackClientConfig is a config for a privileged loopback connection + LoopbackClientConfig *restclient.Config + InsecureServing *apiserver.DeprecatedInsecureServingInfo // nil will disable serving on an insecure port InsecureMetricsServing *apiserver.DeprecatedInsecureServingInfo // non-nil if metrics should be served independently Authentication apiserver.AuthenticationInfo @@ -70,5 +74,7 @@ func (c *Config) Complete() CompletedConfig { c.InsecureMetricsServing.Name = "metrics" } + apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) + return CompletedConfig{&cc} } diff --git a/cmd/kube-scheduler/app/options/BUILD b/cmd/kube-scheduler/app/options/BUILD index ebe68f08340..5806f6cc9b5 100644 --- a/cmd/kube-scheduler/app/options/BUILD +++ b/cmd/kube-scheduler/app/options/BUILD @@ -14,6 +14,7 @@ go_library( "//cmd/kube-scheduler/app/config:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/client/leaderelectionconfig:go_default_library", + "//pkg/master/ports:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/apis/config/scheme:go_default_library", "//pkg/scheduler/apis/config/v1alpha1:go_default_library", diff --git a/cmd/kube-scheduler/app/options/insecure_serving.go b/cmd/kube-scheduler/app/options/insecure_serving.go index 78662f69eb3..db3c7351100 100644 --- a/cmd/kube-scheduler/app/options/insecure_serving.go +++ b/cmd/kube-scheduler/app/options/insecure_serving.go @@ -31,8 +31,8 @@ import ( // CombinedInsecureServingOptions sets up to two insecure listeners for healthz and metrics. The flags // override the ComponentConfig and DeprecatedInsecureServingOptions values for both. type CombinedInsecureServingOptions struct { - Healthz *apiserveroptions.DeprecatedInsecureServingOptions - Metrics *apiserveroptions.DeprecatedInsecureServingOptions + Healthz *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback + Metrics *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback BindPort int // overrides the structs above on ApplyTo, ignored on ApplyToFromLoadedConfig BindAddress string // overrides the structs above on ApplyTo, ignored on ApplyToFromLoadedConfig @@ -60,11 +60,11 @@ func (o *CombinedInsecureServingOptions) applyTo(c *schedulerappconfig.Config, c return err } - if err := o.Healthz.ApplyTo(&c.InsecureServing); err != nil { + if err := o.Healthz.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil { return err } if o.Metrics != nil && (c.ComponentConfig.MetricsBindAddress != c.ComponentConfig.HealthzBindAddress || o.Healthz == nil) { - if err := o.Metrics.ApplyTo(&c.InsecureMetricsServing); err != nil { + if err := o.Metrics.ApplyTo(&c.InsecureMetricsServing, &c.LoopbackClientConfig); err != nil { return err } } @@ -108,7 +108,7 @@ func (o *CombinedInsecureServingOptions) ApplyToFromLoadedConfig(c *schedulerapp return o.applyTo(c, componentConfig) } -func updateAddressFromDeprecatedInsecureServingOptions(addr *string, is *apiserveroptions.DeprecatedInsecureServingOptions) error { +func updateAddressFromDeprecatedInsecureServingOptions(addr *string, is *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback) error { if is == nil { *addr = "" } else { @@ -124,7 +124,7 @@ func updateAddressFromDeprecatedInsecureServingOptions(addr *string, is *apiserv return nil } -func updateDeprecatedInsecureServingOptionsFromAddress(is *apiserveroptions.DeprecatedInsecureServingOptions, addr string) error { +func updateDeprecatedInsecureServingOptionsFromAddress(is *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback, addr string) error { if is == nil { return nil } diff --git a/cmd/kube-scheduler/app/options/insecure_serving_test.go b/cmd/kube-scheduler/app/options/insecure_serving_test.go index 09677b6aa06..28bcb600e60 100644 --- a/cmd/kube-scheduler/app/options/insecure_serving_test.go +++ b/cmd/kube-scheduler/app/options/insecure_serving_test.go @@ -46,8 +46,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -61,7 +61,7 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -79,7 +79,7 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -97,8 +97,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -118,8 +118,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1235", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 0, }, }, @@ -141,8 +141,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindPort: 1236, BindAddress: "1.2.3.4", }, @@ -163,8 +163,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindAddress: "2.3.4.5", BindPort: 1234, }, @@ -185,8 +185,8 @@ func TestOptions_ApplyTo(t *testing.T) { MetricsBindAddress: "1.2.3.4:1234", }, CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{}, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{}, + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), BindAddress: "2.3.4.5", BindPort: 0, }, diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index f35c173687a..fdee98c5982 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -45,6 +45,7 @@ import ( schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/client/leaderelectionconfig" + "k8s.io/kubernetes/pkg/master/ports" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" @@ -56,7 +57,7 @@ type Options struct { // The default values. These are overridden if ConfigFile is set or by values in InsecureServing. ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration - SecureServing *apiserveroptions.SecureServingOptions + SecureServing *apiserveroptions.SecureServingOptionsWithLoopback CombinedInsecureServing *CombinedInsecureServingOptions Authentication *apiserveroptions.DelegatingAuthenticationOptions Authorization *apiserveroptions.DelegatingAuthorizationOptions @@ -85,25 +86,34 @@ func NewOptions() (*Options, error) { o := &Options{ ComponentConfig: *cfg, - SecureServing: nil, // TODO: enable with apiserveroptions.NewSecureServingOptions() + SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: &apiserveroptions.DeprecatedInsecureServingOptions{ + Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{ BindNetwork: "tcp", - }, - Metrics: &apiserveroptions.DeprecatedInsecureServingOptions{ + }).WithLoopback(), + Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{ BindNetwork: "tcp", - }, + }).WithLoopback(), BindPort: hport, BindAddress: hhost, }, - Authentication: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthenticationOptions() - Authorization: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthorizationOptions() + Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), + Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), Deprecated: &DeprecatedOptions{ UseLegacyPolicyConfig: false, PolicyConfigMapNamespace: metav1.NamespaceSystem, }, } + o.Authentication.RemoteKubeConfigFileOptional = true + o.Authorization.RemoteKubeConfigFileOptional = true + o.Authorization.AlwaysAllowPaths = []string{"/healthz"} + + // Set the PairName but leave certificate directory blank to generate in-memory by default + o.SecureServing.ServerCert.CertDirectory = "" + o.SecureServing.ServerCert.PairName = "kube-scheduler" + o.SecureServing.BindPort = ports.KubeSchedulerPort + return o, nil } @@ -173,13 +183,19 @@ func (o *Options) ApplyTo(c *schedulerappconfig.Config) error { } } - if err := o.SecureServing.ApplyTo(&c.SecureServing); err != nil { + if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { return err } - if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { - return err + if o.SecureServing != nil && (o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil) { + if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { + return err + } + if err := o.Authorization.ApplyTo(&c.Authorization); err != nil { + return err + } } - return o.Authorization.ApplyTo(&c.Authorization) + + return nil } // Validate validates all the required options. @@ -200,6 +216,12 @@ func (o *Options) Validate() []error { // Config return a scheduler config object func (o *Options) Config() (*schedulerappconfig.Config, error) { + if o.SecureServing != nil { + if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { + return nil, fmt.Errorf("error creating self-signed certificates: %v", err) + } + } + c := &schedulerappconfig.Config{} if err := o.ApplyTo(c); err != nil { return nil, err diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index b63fac090c4..34cd2404231 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" apiserverconfig "k8s.io/apiserver/pkg/apis/config" + apiserveroptions "k8s.io/apiserver/pkg/server/options" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" ) @@ -175,6 +176,29 @@ users: } return *cfg }(), + SecureServing: (&apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "kube-scheduler", + }, + HTTP2MaxStreamsPerConnection: 47, + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* + }, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -233,6 +257,29 @@ users: cfg.ClientConnection.Kubeconfig = flagKubeconfig return *cfg }(), + SecureServing: (&apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "kube-scheduler", + }, + HTTP2MaxStreamsPerConnection: 47, + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* + }, }, expectedUsername: "flag", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -264,8 +311,32 @@ users: }, }, { - name: "overridden master", - options: &Options{Master: insecureserver.URL}, + name: "overridden master", + options: &Options{ + Master: insecureserver.URL, + SecureServing: (&apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "kube-scheduler", + }, + HTTP2MaxStreamsPerConnection: 47, + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* + }, + }, expectedUsername: "none, http", }, { diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index c88d3ed60bf..2fb574d8b20 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -81,7 +81,7 @@ constraints, affinity and anti-affinity specifications, data locality, inter-wor interference, deadlines, and so on. Workload-specific requirements will be exposed through the API as necessary.`, Run: func(cmd *cobra.Command, args []string) { - if err := run(cmd, args, opts); err != nil { + if err := runCommand(cmd, args, opts); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } @@ -94,8 +94,8 @@ through the API as necessary.`, return cmd } -// run runs the scheduler. -func run(cmd *cobra.Command, args []string, opts *options.Options) error { +// runCommand runs the scheduler. +func runCommand(cmd *cobra.Command, args []string, opts *options.Options) error { verflag.PrintAndExitIfRequested() utilflag.PrintFlags(cmd.Flags()) @@ -136,36 +136,42 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { // Configz registration. if cz, err := configz.New("componentconfig"); err == nil { - cz.Set(c.ComponentConfig) + cz.Set(cc.ComponentConfig) } else { return fmt.Errorf("unable to register configz: %s", err) } + return Run(cc, stopCh) +} + +// Run executes the scheduler based on the given configuration. It only return on error or when stopCh is closed. +func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error { var storageClassInformer storageinformers.StorageClassInformer if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) { - storageClassInformer = c.InformerFactory.Storage().V1().StorageClasses() + storageClassInformer = cc.InformerFactory.Storage().V1().StorageClasses() } // Create the scheduler. - sched, err := scheduler.New(c.Client, - c.InformerFactory.Core().V1().Nodes(), - c.PodInformer, - c.InformerFactory.Core().V1().PersistentVolumes(), - c.InformerFactory.Core().V1().PersistentVolumeClaims(), - c.InformerFactory.Core().V1().ReplicationControllers(), - c.InformerFactory.Apps().V1().ReplicaSets(), - c.InformerFactory.Apps().V1().StatefulSets(), - c.InformerFactory.Core().V1().Services(), - c.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(), + sched, err := scheduler.New(cc.Client, + cc.InformerFactory.Core().V1().Nodes(), + cc.PodInformer, + cc.InformerFactory.Core().V1().PersistentVolumes(), + cc.InformerFactory.Core().V1().PersistentVolumeClaims(), + cc.InformerFactory.Core().V1().ReplicationControllers(), + cc.InformerFactory.Apps().V1().ReplicaSets(), + cc.InformerFactory.Apps().V1().StatefulSets(), + cc.InformerFactory.Core().V1().Services(), + cc.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(), storageClassInformer, - c.Recorder, - c.ComponentConfig.AlgorithmSource, - scheduler.WithName(c.ComponentConfig.SchedulerName), - scheduler.WithHardPodAffinitySymmetricWeight(c.ComponentConfig.HardPodAffinitySymmetricWeight), - scheduler.WithEquivalenceClassCacheEnabled(c.ComponentConfig.EnableContentionProfiling), - scheduler.WithPreemptionDisabled(c.ComponentConfig.DisablePreemption), - scheduler.WithPercentageOfNodesToScore(c.ComponentConfig.PercentageOfNodesToScore), - scheduler.WithBindTimeoutSeconds(*c.ComponentConfig.BindTimeoutSeconds)) + cc.Recorder, + cc.ComponentConfig.AlgorithmSource, + stopCh, + scheduler.WithName(cc.ComponentConfig.SchedulerName), + scheduler.WithHardPodAffinitySymmetricWeight(cc.ComponentConfig.HardPodAffinitySymmetricWeight), + scheduler.WithEquivalenceClassCacheEnabled(cc.ComponentConfig.EnableContentionProfiling), + scheduler.WithPreemptionDisabled(cc.ComponentConfig.DisablePreemption), + scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore), + scheduler.WithBindTimeoutSeconds(*cc.ComponentConfig.BindTimeoutSeconds)) if err != nil { return err } @@ -205,7 +211,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { cc.InformerFactory.WaitForCacheSync(stopCh) controller.WaitForCacheSync("scheduler", stopCh, cc.PodInformer.Informer().HasSynced) - // Prepare a reusable run function. + // Prepare a reusable runCommand function. run := func(ctx context.Context) { sched.Run() <-ctx.Done() @@ -222,7 +228,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { } }() - // If leader election is enabled, run via LeaderElector until done and exit. + // If leader election is enabled, runCommand via LeaderElector until done and exit. if cc.LeaderElection != nil { cc.LeaderElection.Callbacks = leaderelection.LeaderCallbacks{ OnStartedLeading: run, @@ -240,7 +246,7 @@ func run(cmd *cobra.Command, args []string, opts *options.Options) error { return fmt.Errorf("lost lease") } - // Leader election is disabled, so run inline until done. + // Leader election is disabled, so runCommand inline until done. run(ctx) return fmt.Errorf("finished without leader elect") } diff --git a/cmd/kube-scheduler/app/testing/BUILD b/cmd/kube-scheduler/app/testing/BUILD new file mode 100644 index 00000000000..5e4a38bd2ea --- /dev/null +++ b/cmd/kube-scheduler/app/testing/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["testserver.go"], + importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app/testing", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kube-scheduler/app:go_default_library", + "//cmd/kube-scheduler/app/config:go_default_library", + "//cmd/kube-scheduler/app/options:go_default_library", + "//pkg/scheduler/algorithmprovider/defaults:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go new file mode 100644 index 00000000000..0c5250cd443 --- /dev/null +++ b/cmd/kube-scheduler/app/testing/testserver.go @@ -0,0 +1,183 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "time" + + "github.com/spf13/pflag" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubeschedulerconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" + "k8s.io/kubernetes/cmd/kube-scheduler/app/options" + + // import DefaultProvider + _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider/defaults" +) + +// TearDownFunc is to be called to tear down a test server. +type TearDownFunc func() + +// TestServer return values supplied by kube-test-ApiServer +type TestServer struct { + LoopbackClientConfig *restclient.Config // Rest client config using the magic token + Options *options.Options + Config *kubeschedulerconfig.Config + TearDownFn TearDownFunc // TearDown function + TmpDir string // Temp Dir used, by the apiserver +} + +// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie +type Logger interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Logf(format string, args ...interface{}) +} + +// StartTestServer starts a kube-scheduler. A rest client config and a tear-down func, +// and location of the tmpdir are returned. +// +// Note: we return a tear-down func instead of a stop channel because the later will leak temporary +// files that because Golang testing's call to os.Exit will not give a stop channel go routine +// enough time to remove temporary files. +func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { + stopCh := make(chan struct{}) + tearDown := func() { + close(stopCh) + if len(result.TmpDir) != 0 { + os.RemoveAll(result.TmpDir) + } + } + defer func() { + if result.TearDownFn == nil { + tearDown() + } + }() + + result.TmpDir, err = ioutil.TempDir("", "kube-scheduler") + if err != nil { + return result, fmt.Errorf("failed to create temp dir: %v", err) + } + + fs := pflag.NewFlagSet("test", pflag.PanicOnError) + + s, err := options.NewOptions() + if err != nil { + return TestServer{}, err + } + s.AddFlags(fs) + + fs.Parse(customFlags) + + if s.SecureServing.BindPort != 0 { + s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + s.SecureServing.ServerCert.CertDirectory = result.TmpDir + + t.Logf("kube-scheduler will listen securely on port %d...", s.SecureServing.BindPort) + } + + if s.CombinedInsecureServing.BindPort != 0 { + listener, port, err := createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + s.CombinedInsecureServing.BindPort = port + s.CombinedInsecureServing.Healthz.Listener = listener + s.CombinedInsecureServing.Metrics.Listener = listener + t.Logf("kube-scheduler will listen insecurely on port %d...", s.CombinedInsecureServing.BindPort) + } + config, err := s.Config() + if err != nil { + return result, fmt.Errorf("failed to create config from options: %v", err) + } + + errCh := make(chan error) + go func(stopCh <-chan struct{}) { + if err := app.Run(config.Complete(), stopCh); err != nil { + errCh <- err + } + }(stopCh) + + t.Logf("Waiting for /healthz to be ok...") + client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) + if err != nil { + return result, fmt.Errorf("failed to create a client: %v", err) + } + err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { + select { + case err := <-errCh: + return false, err + default: + } + + result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do() + status := 0 + result.StatusCode(&status) + if status == 200 { + return true, nil + } + return false, nil + }) + if err != nil { + return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) + } + + // from here the caller must call tearDown + result.LoopbackClientConfig = config.LoopbackClientConfig + result.Options = s + result.Config = config + result.TearDownFn = tearDown + + return result, nil +} + +// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. +func StartTestServerOrDie(t Logger, flags []string) *TestServer { + result, err := StartTestServer(t, flags) + if err == nil { + return &result + } + + t.Fatalf("failed to launch server: %v", err) + return nil +} + +func createListenerOnFreePort() (net.Listener, int, error) { + ln, err := net.Listen("tcp", ":0") + if err != nil { + return nil, 0, err + } + + // get port + tcpAddr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + ln.Close() + return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) + } + + return ln, tcpAddr.Port, nil +} diff --git a/pkg/master/ports/ports.go b/pkg/master/ports/ports.go index 19207a1012b..23faba1d3ec 100644 --- a/pkg/master/ports/ports.go +++ b/pkg/master/ports/ports.go @@ -23,9 +23,10 @@ const ( // KubeletPort is the default port for the kubelet server on each host machine. // May be overridden by a flag at startup. KubeletPort = 10250 - // SchedulerPort is the default port for the scheduler status server. + // InsecureSchedulerPort is the default port for the scheduler status server. // May be overridden by a flag at startup. - SchedulerPort = 10251 + // Deprecated: use the secure KubeSchedulerPort instead. + InsecureSchedulerPort = 10251 // InsecureKubeControllerManagerPort is the default port for the controller manager status server. // May be overridden by a flag at startup. // Deprecated: use the secure KubeControllerManagerPort instead. @@ -49,4 +50,8 @@ const ( // CloudControllerManagerPort is the default port for the cloud controller manager server. // This value may be overridden by a flag at startup. CloudControllerManagerPort = 10258 + + // KubeSchedulerPort is the default port for the scheduler status server. + // May be overridden by a flag at startup. + KubeSchedulerPort = 10259 ) diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index b9994db1618..78202ee47eb 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -254,7 +254,7 @@ type componentStatusStorage struct { func (s componentStatusStorage) serversToValidate() map[string]*componentstatus.Server { serversToValidate := map[string]*componentstatus.Server{ "controller-manager": {Addr: "127.0.0.1", Port: ports.InsecureKubeControllerManagerPort, Path: "/healthz"}, - "scheduler": {Addr: "127.0.0.1", Port: ports.SchedulerPort, Path: "/healthz"}, + "scheduler": {Addr: "127.0.0.1", Port: ports.InsecureSchedulerPort, Path: "/healthz"}, } for ix, machine := range s.storageFactory.Backends() { diff --git a/pkg/scheduler/apis/config/v1alpha1/defaults.go b/pkg/scheduler/apis/config/v1alpha1/defaults.go index ea608b6b37f..e212dcf4370 100644 --- a/pkg/scheduler/apis/config/v1alpha1/defaults.go +++ b/pkg/scheduler/apis/config/v1alpha1/defaults.go @@ -62,7 +62,7 @@ func SetDefaults_KubeSchedulerConfiguration(obj *kubescedulerconfigv1alpha1.Kube } obj.HealthzBindAddress = net.JoinHostPort(host, port) } else { - obj.HealthzBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.SchedulerPort)) + obj.HealthzBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.InsecureSchedulerPort)) } if host, port, err := net.SplitHostPort(obj.MetricsBindAddress); err == nil { @@ -71,7 +71,7 @@ func SetDefaults_KubeSchedulerConfiguration(obj *kubescedulerconfigv1alpha1.Kube } obj.MetricsBindAddress = net.JoinHostPort(host, port) } else { - obj.MetricsBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.SchedulerPort)) + obj.MetricsBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.InsecureSchedulerPort)) } if len(obj.LeaderElection.LockObjectNamespace) == 0 { diff --git a/pkg/scheduler/factory/BUILD b/pkg/scheduler/factory/BUILD index f9c89f7176e..c3e97c022c5 100644 --- a/pkg/scheduler/factory/BUILD +++ b/pkg/scheduler/factory/BUILD @@ -37,6 +37,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers/apps/v1:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", diff --git a/pkg/scheduler/factory/factory.go b/pkg/scheduler/factory/factory.go index a6a822b0803..078ab6eb5db 100644 --- a/pkg/scheduler/factory/factory.go +++ b/pkg/scheduler/factory/factory.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" appsinformers "k8s.io/client-go/informers/apps/v1" coreinformers "k8s.io/client-go/informers/core/v1" @@ -127,7 +128,7 @@ type Config struct { Recorder record.EventRecorder // Close this to shut down the scheduler. - StopEverything chan struct{} + StopEverything <-chan struct{} // VolumeBinder handles PVC/PV binding for the pod. VolumeBinder *volumebinder.VolumeBinder @@ -200,7 +201,7 @@ type configFactory struct { storageClassLister storagelisters.StorageClassLister // Close this to stop all reflectors - StopEverything chan struct{} + StopEverything <-chan struct{} scheduledPodsHasSynced cache.InformerSynced @@ -253,12 +254,16 @@ type ConfigFactoryArgs struct { DisablePreemption bool PercentageOfNodesToScore int32 BindTimeoutSeconds int64 + StopCh <-chan struct{} } // NewConfigFactory initializes the default implementation of a Configurator. To encourage eventual privatization of the struct type, we only // return the interface. func NewConfigFactory(args *ConfigFactoryArgs) Configurator { - stopEverything := make(chan struct{}) + stopEverything := args.StopCh + if stopEverything == nil { + stopEverything = wait.NeverStop + } schedulerCache := schedulerinternalcache.New(30*time.Second, stopEverything) // storageClassInformer is only enabled through VolumeScheduling feature gate diff --git a/pkg/scheduler/factory/factory_test.go b/pkg/scheduler/factory/factory_test.go index 98f604ca34f..aa68c716fc0 100644 --- a/pkg/scheduler/factory/factory_test.go +++ b/pkg/scheduler/factory/factory_test.go @@ -52,7 +52,9 @@ const ( func TestCreate(t *testing.T) { client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) factory.Create() } @@ -63,7 +65,9 @@ func TestCreateFromConfig(t *testing.T) { var policy schedulerapi.Policy client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) // Pre-register some predicate and priority functions RegisterFitPredicate("PredicateOne", PredicateOne) @@ -101,7 +105,9 @@ func TestCreateFromConfigWithHardPodAffinitySymmetricWeight(t *testing.T) { var policy schedulerapi.Policy client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) // Pre-register some predicate and priority functions RegisterFitPredicate("PredicateOne", PredicateOne) @@ -140,7 +146,9 @@ func TestCreateFromEmptyConfig(t *testing.T) { var policy schedulerapi.Policy client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) configData = []byte(`{}`) if err := runtime.DecodeInto(latestschedulerapi.Codec, configData, &policy); err != nil { @@ -155,7 +163,9 @@ func TestCreateFromEmptyConfig(t *testing.T) { // The predicate/priority from DefaultProvider will be used. func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) RegisterFitPredicate("PredicateOne", PredicateOne) RegisterPriorityFunction("PriorityOne", PriorityOne, 1) @@ -188,7 +198,9 @@ func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { // Empty predicate/priority sets will be used. func TestCreateFromConfigWithEmptyPredicatesOrPriorities(t *testing.T) { client := fake.NewSimpleClientset() - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) RegisterFitPredicate("PredicateOne", PredicateOne) RegisterPriorityFunction("PriorityOne", PriorityOne, 1) @@ -240,7 +252,9 @@ func TestDefaultErrorFunc(t *testing.T) { Spec: apitesting.V1DeepEqualSafePodSpec(), } client := fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testPod}}) - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + defer close(stopCh) + factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) queue := &internalqueue.FIFO{FIFO: cache.NewFIFO(cache.MetaNamespaceKeyFunc)} podBackoff := util.CreatePodBackoff(1*time.Millisecond, 1*time.Second) errFunc := factory.MakeDefaultErrorFunc(podBackoff, queue) @@ -370,7 +384,9 @@ func testBind(binding *v1.Binding, t *testing.T) { func TestInvalidHardPodAffinitySymmetricWeight(t *testing.T) { client := fake.NewSimpleClientset() // factory of "default-scheduler" - factory := newConfigFactory(client, -1) + stopCh := make(chan struct{}) + factory := newConfigFactory(client, -1, stopCh) + defer close(stopCh) _, err := factory.Create() if err == nil { t.Errorf("expected err: invalid hardPodAffinitySymmetricWeight, got nothing") @@ -399,7 +415,9 @@ func TestInvalidFactoryArgs(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - factory := newConfigFactory(client, test.hardPodAffinitySymmetricWeight) + stopCh := make(chan struct{}) + factory := newConfigFactory(client, test.hardPodAffinitySymmetricWeight, stopCh) + defer close(stopCh) _, err := factory.Create() if err == nil { t.Errorf("expected err: %s, got nothing", test.expectErr) @@ -501,7 +519,7 @@ func TestSkipPodUpdate(t *testing.T) { } } -func newConfigFactory(client clientset.Interface, hardPodAffinitySymmetricWeight int32) Configurator { +func newConfigFactory(client clientset.Interface, hardPodAffinitySymmetricWeight int32, stopCh <-chan struct{}) Configurator { informerFactory := informers.NewSharedInformerFactory(client, 0) return NewConfigFactory(&ConfigFactoryArgs{ v1.DefaultSchedulerName, @@ -521,6 +539,7 @@ func newConfigFactory(client clientset.Interface, hardPodAffinitySymmetricWeight disablePodPreemption, schedulerapi.DefaultPercentageOfNodesToScore, bindTimeoutSeconds, + stopCh, }) } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index a30d5b50dd5..4e6a20f8b5c 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -59,12 +59,6 @@ type Scheduler struct { config *factory.Config } -// StopEverything closes the scheduler config's StopEverything channel, to shut -// down the Scheduler. -func (sched *Scheduler) StopEverything() { - close(sched.config.StopEverything) -} - // Cache returns the cache in scheduler for test to check the data in scheduler. func (sched *Scheduler) Cache() schedulerinternalcache.Cache { return sched.config.SchedulerCache @@ -147,6 +141,7 @@ func New(client clientset.Interface, storageClassInformer storageinformers.StorageClassInformer, recorder record.EventRecorder, schedulerAlgorithmSource kubeschedulerconfig.SchedulerAlgorithmSource, + stopCh <-chan struct{}, opts ...func(o *schedulerOptions)) (*Scheduler, error) { options := defaultSchedulerOptions @@ -230,6 +225,7 @@ func New(client clientset.Interface, // Additional tweaks to the config produced by the configurator. config.Recorder = recorder config.DisablePreemption = options.disablePreemption + config.StopEverything = stopCh // Create the scheduler. sched := NewFromConfig(config) return sched, nil diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 699418ac840..b781087baea 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -175,6 +175,8 @@ func TestSchedulerCreation(t *testing.T) { factory.RegisterPriorityFunction("PriorityOne", PriorityOne, 1) factory.RegisterAlgorithmProvider(testSource, sets.NewString("PredicateOne"), sets.NewString("PriorityOne")) + stopCh := make(chan struct{}) + defer close(stopCh) _, err := New(client, informerFactory.Core().V1().Nodes(), factory.NewPodInformer(client, 0), @@ -188,6 +190,7 @@ func TestSchedulerCreation(t *testing.T) { informerFactory.Storage().V1().StorageClasses(), eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "scheduler"}), kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &testSource}, + stopCh, WithBindTimeoutSeconds(defaultBindTimeout)) if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go index e8e3d7feb3f..f1cc4430b8b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go @@ -150,11 +150,11 @@ func (s *DeprecatedInsecureServingOptionsWithLoopback) ApplyTo(insecureServingIn secureLoopbackClientConfig, err := (*insecureServingInfo).NewLoopbackClientConfig() switch { // if we failed and there's no fallback loopback client config, we need to fail - case err != nil && secureLoopbackClientConfig == nil: + case err != nil && *loopbackClientConfig == nil: return err // if we failed, but we already have a fallback loopback client config (usually insecure), allow it - case err != nil && secureLoopbackClientConfig != nil: + case err != nil && *loopbackClientConfig != nil: default: *loopbackClientConfig = secureLoopbackClientConfig diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go index ac9635d2dd3..7f19206425e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go @@ -63,11 +63,11 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se secureLoopbackClientConfig, err := (*secureServingInfo).NewLoopbackClientConfig(uuid.NewRandom().String(), certPem) switch { // if we failed and there's no fallback loopback client config, we need to fail - case err != nil && secureLoopbackClientConfig == nil: + case err != nil && *loopbackClientConfig == nil: return err // if we failed, but we already have a fallback loopback client config (usually insecure), allow it - case err != nil && secureLoopbackClientConfig != nil: + case err != nil && *loopbackClientConfig != nil: default: *loopbackClientConfig = secureLoopbackClientConfig diff --git a/test/e2e/apps/daemon_restart.go b/test/e2e/apps/daemon_restart.go index dfb2c6a1a2c..e37a2bda03f 100644 --- a/test/e2e/apps/daemon_restart.go +++ b/test/e2e/apps/daemon_restart.go @@ -281,7 +281,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() { // Requires master ssh access. framework.SkipUnlessProviderIs("gce", "aws") restarter := NewRestartConfig( - framework.GetMasterHost(), "kube-scheduler", ports.SchedulerPort, restartPollInterval, restartTimeout) + framework.GetMasterHost(), "kube-scheduler", ports.InsecureSchedulerPort, restartPollInterval, restartTimeout) // Create pods while the scheduler is down and make sure the scheduler picks them up by // scaling the rc to the same size. diff --git a/test/e2e/framework/metrics/metrics_grabber.go b/test/e2e/framework/metrics/metrics_grabber.go index dd95dea3eb6..a9851b2cc8c 100644 --- a/test/e2e/framework/metrics/metrics_grabber.go +++ b/test/e2e/framework/metrics/metrics_grabber.go @@ -127,7 +127,7 @@ func (g *MetricsGrabber) GrabFromScheduler() (SchedulerMetrics, error) { if !g.registeredMaster { return SchedulerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping Scheduler's metrics gathering.") } - output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, ports.SchedulerPort) + output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, ports.InsecureSchedulerPort) if err != nil { return SchedulerMetrics{}, err } diff --git a/test/e2e/framework/metrics_util.go b/test/e2e/framework/metrics_util.go index 1ef6b37268e..9f651df0d37 100644 --- a/test/e2e/framework/metrics_util.go +++ b/test/e2e/framework/metrics_util.go @@ -611,7 +611,7 @@ func sendRestRequestToScheduler(c clientset.Interface, op string) (string, error Context(ctx). Namespace(metav1.NamespaceSystem). Resource("pods"). - Name(fmt.Sprintf("kube-scheduler-%v:%v", TestContext.CloudConfig.MasterName, ports.SchedulerPort)). + Name(fmt.Sprintf("kube-scheduler-%v:%v", TestContext.CloudConfig.MasterName, ports.InsecureSchedulerPort)). SubResource("proxy"). Suffix("metrics"). Do().Raw() diff --git a/test/e2e/network/firewall.go b/test/e2e/network/firewall.go index d46ff96c352..ee3839f16d2 100644 --- a/test/e2e/network/firewall.go +++ b/test/e2e/network/firewall.go @@ -180,7 +180,7 @@ var _ = SIGDescribe("Firewall rule", func() { masterAddresses := framework.GetAllMasterAddresses(cs) for _, masterAddress := range masterAddresses { assertNotReachableHTTPTimeout(masterAddress, ports.InsecureKubeControllerManagerPort, gce.FirewallTestTcpTimeout) - assertNotReachableHTTPTimeout(masterAddress, ports.SchedulerPort, gce.FirewallTestTcpTimeout) + assertNotReachableHTTPTimeout(masterAddress, ports.InsecureSchedulerPort, gce.FirewallTestTcpTimeout) } assertNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletPort, gce.FirewallTestTcpTimeout) assertNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletReadOnlyPort, gce.FirewallTestTcpTimeout) diff --git a/test/integration/BUILD b/test/integration/BUILD index 1fe6fa9d2ce..d024c667e5c 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -41,7 +41,6 @@ filegroup( "//test/integration/benchmark/jsonify:all-srcs", "//test/integration/client:all-srcs", "//test/integration/configmap:all-srcs", - "//test/integration/controllermanager:all-srcs", "//test/integration/cronjob:all-srcs", "//test/integration/daemonset:all-srcs", "//test/integration/defaulttolerationseconds:all-srcs", @@ -66,6 +65,7 @@ filegroup( "//test/integration/scheduler_perf:all-srcs", "//test/integration/secrets:all-srcs", "//test/integration/serviceaccount:all-srcs", + "//test/integration/serving:all-srcs", "//test/integration/statefulset:all-srcs", "//test/integration/storageclasses:all-srcs", "//test/integration/tls:all-srcs", diff --git a/test/integration/scheduler/extender_test.go b/test/integration/scheduler/extender_test.go index aa6fad56484..2afef4b0d0f 100644 --- a/test/integration/scheduler/extender_test.go +++ b/test/integration/scheduler/extender_test.go @@ -349,7 +349,7 @@ func TestSchedulerExtender(t *testing.T) { } policy.APIVersion = "v1" - context = initTestScheduler(t, context, nil, false, &policy) + context = initTestScheduler(t, context, false, &policy) defer cleanupTest(t, context) DoTestPodScheduling(context.ns, t, clientSet) diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 3151768454a..3f7938bb5ff 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -522,7 +522,10 @@ func TestMultiScheduler(t *testing.T) { informerFactory2 := informers.NewSharedInformerFactory(context.clientSet, 0) podInformer2 := factory.NewPodInformer(context.clientSet, 0) - schedulerConfigFactory2 := createConfiguratorWithPodInformer(fooScheduler, clientSet2, podInformer2, informerFactory2) + stopCh := make(chan struct{}) + defer close(stopCh) + + schedulerConfigFactory2 := createConfiguratorWithPodInformer(fooScheduler, clientSet2, podInformer2, informerFactory2, stopCh) schedulerConfig2, err := schedulerConfigFactory2.Create() if err != nil { t.Errorf("Couldn't create scheduler config: %v", err) @@ -530,12 +533,11 @@ func TestMultiScheduler(t *testing.T) { eventBroadcaster2 := record.NewBroadcaster() schedulerConfig2.Recorder = eventBroadcaster2.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: fooScheduler}) eventBroadcaster2.StartRecordingToSink(&clientv1core.EventSinkImpl{Interface: clientSet2.CoreV1().Events("")}) - go podInformer2.Informer().Run(schedulerConfig2.StopEverything) - informerFactory2.Start(schedulerConfig2.StopEverything) + go podInformer2.Informer().Run(stopCh) + informerFactory2.Start(stopCh) sched2, _ := scheduler.NewFromConfigurator(&scheduler.FakeConfigurator{Config: schedulerConfig2}, nil...) sched2.Run() - defer close(schedulerConfig2.StopEverything) // 6. **check point-2**: // - testPodWithAnnotationFitsFoo should be scheduled diff --git a/test/integration/scheduler/taint_test.go b/test/integration/scheduler/taint_test.go index 89189fc1555..e6964760ec4 100644 --- a/test/integration/scheduler/taint_test.go +++ b/test/integration/scheduler/taint_test.go @@ -85,13 +85,10 @@ func TestTaintNodeByCondition(t *testing.T) { admission.SetExternalKubeClientSet(externalClientset) admission.SetExternalKubeInformerFactory(externalInformers) - controllerCh := make(chan struct{}) - defer close(controllerCh) - // Apply feature gates to enable TaintNodesByCondition algorithmprovider.ApplyFeatureGates() - context = initTestScheduler(t, context, controllerCh, false, nil) + context = initTestScheduler(t, context, false, nil) cs := context.clientSet informers := context.informerFactory nsName := context.ns.Name @@ -120,13 +117,13 @@ func TestTaintNodeByCondition(t *testing.T) { t.Errorf("Failed to create node controller: %v", err) return } - go nc.Run(controllerCh) + go nc.Run(context.stopCh) // Waiting for all controller sync. - externalInformers.Start(controllerCh) - externalInformers.WaitForCacheSync(controllerCh) - informers.Start(controllerCh) - informers.WaitForCacheSync(controllerCh) + externalInformers.Start(context.stopCh) + externalInformers.WaitForCacheSync(context.stopCh) + informers.Start(context.stopCh) + informers.WaitForCacheSync(context.stopCh) // ------------------------------------------- // Test TaintNodeByCondition feature. diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index ddc624a050f..4ff1d81f87b 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -65,6 +65,7 @@ type TestContext struct { schedulerConfigFactory factory.Configurator schedulerConfig *factory.Config scheduler *scheduler.Scheduler + stopCh chan struct{} } // createConfiguratorWithPodInformer creates a configurator for scheduler. @@ -73,6 +74,7 @@ func createConfiguratorWithPodInformer( clientSet clientset.Interface, podInformer coreinformers.PodInformer, informerFactory informers.SharedInformerFactory, + stopCh <-chan struct{}, ) factory.Configurator { return factory.NewConfigFactory(&factory.ConfigFactoryArgs{ SchedulerName: schedulerName, @@ -92,13 +94,16 @@ func createConfiguratorWithPodInformer( DisablePreemption: false, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, BindTimeoutSeconds: 600, + StopCh: stopCh, }) } // initTestMasterAndScheduler initializes a test environment and creates a master with default // configuration. func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface) *TestContext { - var context TestContext + context := TestContext{ + stopCh: make(chan struct{}), + } // 1. Create master h := &framework.MasterHolder{Initialized: make(chan struct{})} @@ -138,13 +143,12 @@ func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface func initTestScheduler( t *testing.T, context *TestContext, - controllerCh chan struct{}, setPodInformer bool, policy *schedulerapi.Policy, ) *TestContext { // Pod preemption is enabled by default scheduler configuration, but preemption only happens when PodPriority // feature gate is enabled at the same time. - return initTestSchedulerWithOptions(t, context, controllerCh, setPodInformer, policy, false, false, time.Second) + return initTestSchedulerWithOptions(t, context, setPodInformer, policy, false, false, time.Second) } // initTestSchedulerWithOptions initializes a test environment and creates a scheduler with default @@ -152,7 +156,6 @@ func initTestScheduler( func initTestSchedulerWithOptions( t *testing.T, context *TestContext, - controllerCh chan struct{}, setPodInformer bool, policy *schedulerapi.Policy, disablePreemption bool, @@ -179,7 +182,7 @@ func initTestSchedulerWithOptions( } context.schedulerConfigFactory = createConfiguratorWithPodInformer( - v1.DefaultSchedulerName, context.clientSet, podInformer, context.informerFactory) + v1.DefaultSchedulerName, context.clientSet, podInformer, context.informerFactory, context.stopCh) var err error @@ -193,11 +196,6 @@ func initTestSchedulerWithOptions( t.Fatalf("Couldn't create scheduler config: %v", err) } - // set controllerCh if provided. - if controllerCh != nil { - context.schedulerConfig.StopEverything = controllerCh - } - // set DisablePreemption option context.schedulerConfig.DisablePreemption = disablePreemption @@ -252,21 +250,21 @@ func initDisruptionController(context *TestContext) *disruption.DisruptionContro // initTest initializes a test environment and creates master and scheduler with default // configuration. func initTest(t *testing.T, nsPrefix string) *TestContext { - return initTestScheduler(t, initTestMaster(t, nsPrefix, nil), nil, true, nil) + return initTestScheduler(t, initTestMaster(t, nsPrefix, nil), true, nil) } // initTestDisablePreemption initializes a test environment and creates master and scheduler with default // configuration but with pod preemption disabled. func initTestDisablePreemption(t *testing.T, nsPrefix string) *TestContext { return initTestSchedulerWithOptions( - t, initTestMaster(t, nsPrefix, nil), nil, true, nil, true, false, time.Second) + t, initTestMaster(t, nsPrefix, nil), true, nil, true, false, time.Second) } // cleanupTest deletes the scheduler and the test namespace. It should be called // at the end of a test. func cleanupTest(t *testing.T, context *TestContext) { // Kill the scheduler. - close(context.schedulerConfig.StopEverything) + close(context.stopCh) // Cleanup nodes. context.clientSet.CoreV1().Nodes().DeleteCollection(nil, metav1.ListOptions{}) framework.DeleteTestingNamespace(context.ns, context.httpServer, t) diff --git a/test/integration/scheduler/volume_binding_test.go b/test/integration/scheduler/volume_binding_test.go index e82b3562be2..9e694f8785a 100644 --- a/test/integration/scheduler/volume_binding_test.go +++ b/test/integration/scheduler/volume_binding_test.go @@ -901,9 +901,7 @@ func setupCluster(t *testing.T, nsName string, numberOfNodes int, features map[s // Set feature gates utilfeature.DefaultFeatureGate.SetFromMap(features) - controllerCh := make(chan struct{}) - - context := initTestSchedulerWithOptions(t, initTestMaster(t, nsName, nil), controllerCh, false, nil, false, disableEquivalenceCache, resyncPeriod) + context := initTestSchedulerWithOptions(t, initTestMaster(t, nsName, nil), false, nil, false, disableEquivalenceCache, resyncPeriod) clientset := context.clientSet ns := context.ns.Name @@ -912,10 +910,10 @@ func setupCluster(t *testing.T, nsName string, numberOfNodes int, features map[s if err != nil { t.Fatalf("Failed to create PV controller: %v", err) } - go ctrl.Run(controllerCh) + go ctrl.Run(context.stopCh) // Start informer factory after all controllers are configured and running. - informerFactory.Start(controllerCh) - informerFactory.WaitForCacheSync(controllerCh) + informerFactory.Start(context.stopCh) + informerFactory.WaitForCacheSync(context.stopCh) // Create shared objects // Create nodes @@ -936,7 +934,7 @@ func setupCluster(t *testing.T, nsName string, numberOfNodes int, features map[s return &testConfig{ client: clientset, ns: ns, - stop: controllerCh, + stop: context.stopCh, teardown: func() { deleteTestObjects(clientset, ns, nil) cleanupTest(t, context) diff --git a/test/integration/controllermanager/BUILD b/test/integration/serving/BUILD similarity index 95% rename from test/integration/controllermanager/BUILD rename to test/integration/serving/BUILD index 1ebcf099d89..24fe855f3ee 100644 --- a/test/integration/controllermanager/BUILD +++ b/test/integration/serving/BUILD @@ -20,6 +20,7 @@ go_test( "//cmd/cloud-controller-manager/app/testing:go_default_library", "//cmd/kube-apiserver/app/testing:go_default_library", "//cmd/kube-controller-manager/app/testing:go_default_library", + "//cmd/kube-scheduler/app/testing:go_default_library", "//pkg/cloudprovider/providers/fake:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/test/integration/controllermanager/main_test.go b/test/integration/serving/main_test.go similarity index 96% rename from test/integration/controllermanager/main_test.go rename to test/integration/serving/main_test.go index 0ef7244c3c8..5675067c7aa 100644 --- a/test/integration/controllermanager/main_test.go +++ b/test/integration/serving/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllermanager +package serving import ( "testing" diff --git a/test/integration/controllermanager/serving_test.go b/test/integration/serving/serving_test.go similarity index 88% rename from test/integration/controllermanager/serving_test.go rename to test/integration/serving/serving_test.go index da315adefff..668e1c26454 100644 --- a/test/integration/controllermanager/serving_test.go +++ b/test/integration/serving/serving_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllermanager +package serving import ( "crypto/tls" @@ -33,15 +33,16 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/kubernetes" - cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider" cloudctrlmgrtesting "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" + kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing" "k8s.io/kubernetes/pkg/cloudprovider/providers/fake" "k8s.io/kubernetes/test/integration/framework" ) -type controllerManagerTester interface { +type componentTester interface { StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) } @@ -65,7 +66,17 @@ func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err } -func TestControllerManagerServing(t *testing.T) { +type kubeSchedulerTester struct{} + +func (kubeSchedulerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := kubeschedulertesting.StartTestServer(t, customFlags) + if err != nil { + return nil, nil, nil, nil, err + } + return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err +} + +func TestComponentSecureServingAndAuth(t *testing.T) { if !cloudprovider.IsCloudProvider("fake") { cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory) } @@ -188,20 +199,21 @@ users: tests := []struct { name string - tester controllerManagerTester + tester componentTester extraFlags []string }{ {"kube-controller-manager", kubeControllerManagerTester{}, nil}, {"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}}, + {"kube-scheduler", kubeSchedulerTester{}, nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - testControllerManager(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) + testComponent(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) }) } } -func testControllerManager(t *testing.T, tester controllerManagerTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { +func testComponent(t *testing.T, tester componentTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { tests := []struct { name string flags []string @@ -228,8 +240,7 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", true, false, intPtr(http.StatusOK), nil}, - {"/metrics without auhn/z", []string{ - "--kubeconfig", kubeconfig, + {"/metrics without authn/authz", []string{ "--kubeconfig", kubeconfig, "--leader-elect=false", "--port=10253", @@ -297,7 +308,7 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt") serverCert, err := ioutil.ReadFile(serverCertPath) if err != nil { - t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err) + t.Fatalf("Failed to read component server cert %q: %v", serverCertPath, err) } pool.AppendCertsFromPEM(serverCert) tr := &http.Transport{ @@ -316,13 +327,13 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon } r, err := client.Do(req) if err != nil { - t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) + t.Fatalf("failed to GET %s from component: %v", tt.path, err) } body, err := ioutil.ReadAll(r.Body) defer r.Body.Close() if got, expected := r.StatusCode, *tt.wantSecureCode; got != expected { - t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body)) + t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) } } @@ -332,12 +343,12 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon url := fmt.Sprintf("http://%s%s", insecureInfo.Listener.Addr().String(), tt.path) r, err := http.Get(url) if err != nil { - t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) + t.Fatalf("failed to GET %s from component: %v", tt.path, err) } body, err := ioutil.ReadAll(r.Body) defer r.Body.Close() if got, expected := r.StatusCode, *tt.wantInsecureCode; got != expected { - t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body)) + t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) } } }) diff --git a/test/integration/util/util.go b/test/integration/util/util.go index fa39562d513..d79c17641cd 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -66,7 +66,8 @@ func StartScheduler(clientSet clientset.Interface) (factory.Configurator, Shutdo evtWatch := evtBroadcaster.StartRecordingToSink(&clientv1core.EventSinkImpl{ Interface: clientSet.CoreV1().Events("")}) - schedulerConfigurator := createSchedulerConfigurator(clientSet, informerFactory) + stopCh := make(chan struct{}) + schedulerConfigurator := createSchedulerConfigurator(clientSet, informerFactory, stopCh) sched, err := scheduler.NewFromConfigurator(schedulerConfigurator, func(conf *factory.Config) { conf.Recorder = evtBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "scheduler"}) @@ -75,16 +76,13 @@ func StartScheduler(clientSet clientset.Interface) (factory.Configurator, Shutdo glog.Fatalf("Error creating scheduler: %v", err) } - stop := make(chan struct{}) - informerFactory.Start(stop) - + informerFactory.Start(stopCh) sched.Run() shutdownFunc := func() { glog.Infof("destroying scheduler") evtWatch.Stop() - sched.StopEverything() - close(stop) + close(stopCh) glog.Infof("destroyed scheduler") } return schedulerConfigurator, shutdownFunc @@ -94,6 +92,7 @@ func StartScheduler(clientSet clientset.Interface) (factory.Configurator, Shutdo func createSchedulerConfigurator( clientSet clientset.Interface, informerFactory informers.SharedInformerFactory, + stopCh <-chan struct{}, ) factory.Configurator { // Enable EnableEquivalenceClassCache for all integration tests. utilfeature.DefaultFeatureGate.Set("EnableEquivalenceClassCache=true") @@ -115,5 +114,6 @@ func createSchedulerConfigurator( EnableEquivalenceClassCache: utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache), DisablePreemption: false, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, + StopCh: stopCh, }) }