From ea8e1e01c5722a7d179eabd2654406adf7951635 Mon Sep 17 00:00:00 2001 From: Jun Gong Date: Tue, 21 May 2019 18:27:37 +0800 Subject: [PATCH 1/4] Add support for writing out of tree custom scheduler plugins --- cmd/kube-scheduler/app/server.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 58745fa9fda..65e5b140f6a 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -58,6 +58,8 @@ import ( "k8s.io/klog" ) +var frameworkRegistry = framework.NewRegistry() + // NewSchedulerCommand creates a *cobra.Command object with default parameters func NewSchedulerCommand() *cobra.Command { opts, err := options.NewOptions() @@ -175,7 +177,7 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error cc.Recorder, cc.ComponentConfig.AlgorithmSource, stopCh, - framework.NewRegistry(), + frameworkRegistry, cc.ComponentConfig.Plugins, cc.ComponentConfig.PluginConfig, scheduler.WithName(cc.ComponentConfig.SchedulerName), @@ -326,3 +328,9 @@ func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, s } return pathRecorderMux } + +// RegisterFrameworkPlugin adds a new plugin to the registry. If a plugin with the same name +// exists, it returns an error. +func RegisterFrameworkPlugin(name string, factory framework.PluginFactory) error { + return frameworkRegistry.Register(name, factory) +} From e9793c8bdbc3e71d9d3203272e9f86b5e029d878 Mon Sep 17 00:00:00 2001 From: Jun Gong Date: Mon, 24 Jun 2019 11:09:25 +0800 Subject: [PATCH 2/4] Pass registry parameter to scheduler instead of global singleton --- cmd/kube-scheduler/app/config/BUILD | 1 + cmd/kube-scheduler/app/config/config.go | 7 +++++++ cmd/kube-scheduler/app/server.go | 23 ++++++++++++----------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/kube-scheduler/app/config/BUILD b/cmd/kube-scheduler/app/config/BUILD index 877ceef1e05..851b3ab3f75 100644 --- a/cmd/kube-scheduler/app/config/BUILD +++ b/cmd/kube-scheduler/app/config/BUILD @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 2ad5ae9bf34..02243f5aa0d 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/record" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) // Config has all the context to run a Scheduler @@ -51,6 +52,9 @@ type Config struct { // LeaderElection is optional. LeaderElection *leaderelection.LeaderElectionConfig + + // Registry is a collection of all available plugins. + Registry framework.Registry } type completedConfig struct { @@ -73,6 +77,9 @@ func (c *Config) Complete() CompletedConfig { if c.InsecureMetricsServing != nil { c.InsecureMetricsServing.Name = "metrics" } + if c.Registry == nil { + c.Registry = framework.NewRegistry() + } apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 65e5b140f6a..545450e49eb 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -58,10 +58,13 @@ import ( "k8s.io/klog" ) -var frameworkRegistry = framework.NewRegistry() - // NewSchedulerCommand creates a *cobra.Command object with default parameters func NewSchedulerCommand() *cobra.Command { + return NewSchedulerCommandWithRegistry(nil) +} + +// NewSchedulerCommandWithRegistry creates a *cobra.Command object with registry and default parameters +func NewSchedulerCommandWithRegistry(registry framework.Registry) *cobra.Command { opts, err := options.NewOptions() if err != nil { klog.Fatalf("unable to initialize command options: %v", err) @@ -77,7 +80,7 @@ constraints, affinity and anti-affinity specifications, data locality, inter-wor interference, deadlines, and so on. Workload-specific requirements will be exposed through the API as necessary.`, Run: func(cmd *cobra.Command, args []string) { - if err := runCommand(cmd, args, opts); err != nil { + if err := runCommand(cmd, args, opts, registry); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } @@ -108,7 +111,7 @@ through the API as necessary.`, } // runCommand runs the scheduler. -func runCommand(cmd *cobra.Command, args []string, opts *options.Options) error { +func runCommand(cmd *cobra.Command, args []string, opts *options.Options, registry framework.Registry) error { verflag.PrintAndExitIfRequested() utilflag.PrintFlags(cmd.Flags()) @@ -137,6 +140,10 @@ func runCommand(cmd *cobra.Command, args []string, opts *options.Options) error stopCh := make(chan struct{}) + if registry != nil { + c.Registry = registry + } + // Get the completed config cc := c.Complete() @@ -177,7 +184,7 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error cc.Recorder, cc.ComponentConfig.AlgorithmSource, stopCh, - frameworkRegistry, + cc.Registry, cc.ComponentConfig.Plugins, cc.ComponentConfig.PluginConfig, scheduler.WithName(cc.ComponentConfig.SchedulerName), @@ -328,9 +335,3 @@ func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, s } return pathRecorderMux } - -// RegisterFrameworkPlugin adds a new plugin to the registry. If a plugin with the same name -// exists, it returns an error. -func RegisterFrameworkPlugin(name string, factory framework.PluginFactory) error { - return frameworkRegistry.Register(name, factory) -} From 107e98945411f8f75f789661246237890cf19d92 Mon Sep 17 00:00:00 2001 From: Jun Gong Date: Tue, 25 Jun 2019 16:09:54 +0800 Subject: [PATCH 3/4] Use WithOption to implment scheduler register --- cmd/kube-scheduler/app/config/BUILD | 1 - cmd/kube-scheduler/app/config/config.go | 7 ----- cmd/kube-scheduler/app/server.go | 39 +++++++++++++++---------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/cmd/kube-scheduler/app/config/BUILD b/cmd/kube-scheduler/app/config/BUILD index 851b3ab3f75..877ceef1e05 100644 --- a/cmd/kube-scheduler/app/config/BUILD +++ b/cmd/kube-scheduler/app/config/BUILD @@ -7,7 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/scheduler/apis/config:go_default_library", - "//pkg/scheduler/framework/v1alpha1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 02243f5aa0d..2ad5ae9bf34 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -26,7 +26,6 @@ import ( "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/record" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) // Config has all the context to run a Scheduler @@ -52,9 +51,6 @@ type Config struct { // LeaderElection is optional. LeaderElection *leaderelection.LeaderElectionConfig - - // Registry is a collection of all available plugins. - Registry framework.Registry } type completedConfig struct { @@ -77,9 +73,6 @@ func (c *Config) Complete() CompletedConfig { if c.InsecureMetricsServing != nil { c.InsecureMetricsServing.Name = "metrics" } - if c.Registry == nil { - c.Registry = framework.NewRegistry() - } apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 545450e49eb..318179a7363 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -58,13 +58,11 @@ import ( "k8s.io/klog" ) -// NewSchedulerCommand creates a *cobra.Command object with default parameters -func NewSchedulerCommand() *cobra.Command { - return NewSchedulerCommandWithRegistry(nil) -} +// PluginOption is an function type used for dealing with scheduler registry. +type PluginOption func(framework.Registry) error -// NewSchedulerCommandWithRegistry creates a *cobra.Command object with registry and default parameters -func NewSchedulerCommandWithRegistry(registry framework.Registry) *cobra.Command { +// NewSchedulerCommand creates a *cobra.Command object with default parameters and pluginOptions +func NewSchedulerCommand(pluginOptions ...PluginOption) *cobra.Command { opts, err := options.NewOptions() if err != nil { klog.Fatalf("unable to initialize command options: %v", err) @@ -80,7 +78,7 @@ constraints, affinity and anti-affinity specifications, data locality, inter-wor interference, deadlines, and so on. Workload-specific requirements will be exposed through the API as necessary.`, Run: func(cmd *cobra.Command, args []string) { - if err := runCommand(cmd, args, opts, registry); err != nil { + if err := runCommand(cmd, args, opts, pluginOptions...); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } @@ -111,7 +109,7 @@ through the API as necessary.`, } // runCommand runs the scheduler. -func runCommand(cmd *cobra.Command, args []string, opts *options.Options, registry framework.Registry) error { +func runCommand(cmd *cobra.Command, args []string, opts *options.Options, pluginOptions ...PluginOption) error { verflag.PrintAndExitIfRequested() utilflag.PrintFlags(cmd.Flags()) @@ -139,11 +137,6 @@ func runCommand(cmd *cobra.Command, args []string, opts *options.Options, regist } stopCh := make(chan struct{}) - - if registry != nil { - c.Registry = registry - } - // Get the completed config cc := c.Complete() @@ -161,14 +154,21 @@ func runCommand(cmd *cobra.Command, args []string, opts *options.Options, regist return fmt.Errorf("unable to register configz: %s", err) } - return Run(cc, stopCh) + return Run(cc, stopCh, pluginOptions...) } // Run executes the scheduler based on the given configuration. It only return on error or when stopCh is closed. -func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error { +func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}, pluginOptions ...PluginOption) error { // To help debugging, immediately log version klog.V(1).Infof("Starting Kubernetes Scheduler version %+v", version.Get()) + registry := framework.NewRegistry() + for _, option := range pluginOptions { + if err := option(registry); err != nil { + return err + } + } + // Create the scheduler. sched, err := scheduler.New(cc.Client, cc.InformerFactory.Core().V1().Nodes(), @@ -184,7 +184,7 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error cc.Recorder, cc.ComponentConfig.AlgorithmSource, stopCh, - cc.Registry, + registry, cc.ComponentConfig.Plugins, cc.ComponentConfig.PluginConfig, scheduler.WithName(cc.ComponentConfig.SchedulerName), @@ -335,3 +335,10 @@ func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, s } return pathRecorderMux } + +// WithPlugin creates a PluginOption based on plugin name and factory. +func WithPlugin(name string, factory framework.PluginFactory) PluginOption { + return func(registry framework.Registry) error { + return registry.Register(name, factory) + } +} From 5828223e756d60d9a5ba3798b44c3dd2ffd64edf Mon Sep 17 00:00:00 2001 From: Jun Gong Date: Thu, 11 Jul 2019 17:39:48 +0800 Subject: [PATCH 4/4] Rename PluginOption to Option --- cmd/kube-scheduler/app/server.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 318179a7363..0da0b8d2303 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -58,11 +58,11 @@ import ( "k8s.io/klog" ) -// PluginOption is an function type used for dealing with scheduler registry. -type PluginOption func(framework.Registry) error +// Option configures a framework.Registry. +type Option func(framework.Registry) error -// NewSchedulerCommand creates a *cobra.Command object with default parameters and pluginOptions -func NewSchedulerCommand(pluginOptions ...PluginOption) *cobra.Command { +// NewSchedulerCommand creates a *cobra.Command object with default parameters and registryOptions +func NewSchedulerCommand(registryOptions ...Option) *cobra.Command { opts, err := options.NewOptions() if err != nil { klog.Fatalf("unable to initialize command options: %v", err) @@ -78,7 +78,7 @@ constraints, affinity and anti-affinity specifications, data locality, inter-wor interference, deadlines, and so on. Workload-specific requirements will be exposed through the API as necessary.`, Run: func(cmd *cobra.Command, args []string) { - if err := runCommand(cmd, args, opts, pluginOptions...); err != nil { + if err := runCommand(cmd, args, opts, registryOptions...); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } @@ -109,7 +109,7 @@ through the API as necessary.`, } // runCommand runs the scheduler. -func runCommand(cmd *cobra.Command, args []string, opts *options.Options, pluginOptions ...PluginOption) error { +func runCommand(cmd *cobra.Command, args []string, opts *options.Options, registryOptions ...Option) error { verflag.PrintAndExitIfRequested() utilflag.PrintFlags(cmd.Flags()) @@ -154,16 +154,16 @@ func runCommand(cmd *cobra.Command, args []string, opts *options.Options, plugin return fmt.Errorf("unable to register configz: %s", err) } - return Run(cc, stopCh, pluginOptions...) + return Run(cc, stopCh, registryOptions...) } // Run executes the scheduler based on the given configuration. It only return on error or when stopCh is closed. -func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}, pluginOptions ...PluginOption) error { +func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}, registryOptions ...Option) error { // To help debugging, immediately log version klog.V(1).Infof("Starting Kubernetes Scheduler version %+v", version.Get()) registry := framework.NewRegistry() - for _, option := range pluginOptions { + for _, option := range registryOptions { if err := option(registry); err != nil { return err } @@ -336,8 +336,8 @@ func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, s return pathRecorderMux } -// WithPlugin creates a PluginOption based on plugin name and factory. -func WithPlugin(name string, factory framework.PluginFactory) PluginOption { +// WithPlugin creates an Option based on plugin name and factory. +func WithPlugin(name string, factory framework.PluginFactory) Option { return func(registry framework.Registry) error { return registry.Register(name, factory) }