From e7f67b1a63c048fac0b89c0f355206795a9767a7 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Mon, 29 Mar 2021 12:27:27 -0700 Subject: [PATCH] Surface kube config in scheduler framework handle --- cmd/kube-scheduler/app/config/config.go | 1 + cmd/kube-scheduler/app/options/options.go | 1 + cmd/kube-scheduler/app/server.go | 1 + pkg/scheduler/factory.go | 5 ++++- pkg/scheduler/framework/interface.go | 4 ++++ pkg/scheduler/framework/runtime/framework.go | 16 +++++++++++++++ pkg/scheduler/scheduler.go | 10 ++++++++++ pkg/scheduler/scheduler_test.go | 6 ++++-- test/integration/scheduler/framework_test.go | 2 +- test/integration/scheduler_perf/util.go | 2 +- test/integration/util/util.go | 21 +++++++++++--------- 11 files changed, 55 insertions(+), 14 deletions(-) diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 9ee3dcb0fc9..2b36c1c880d 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -41,6 +41,7 @@ type Config struct { SecureServing *apiserver.SecureServingInfo Client clientset.Interface + KubeConfig *restclient.Config InformerFactory informers.SharedInformerFactory //lint:ignore SA1019 this deprecated field still needs to be used for now. It will be removed once the migration is done. diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 87ff1424ec2..3e61a46f8ee 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -289,6 +289,7 @@ func (o *Options) Config() (*schedulerappconfig.Config, error) { } c.Client = client + c.KubeConfig = kubeConfig c.InformerFactory = scheduler.NewInformerFactory(client, 0) c.LeaderElection = leaderElectionConfig diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 8bc747688a3..25b08ebac19 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -323,6 +323,7 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions cc.InformerFactory, recorderFactory, ctx.Done(), + scheduler.WithKubeConfig(cc.KubeConfig), scheduler.WithProfiles(cc.ComponentConfig.Profiles...), scheduler.WithAlgorithmSource(cc.ComponentConfig.AlgorithmSource), scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore), diff --git a/pkg/scheduler/factory.go b/pkg/scheduler/factory.go index 12e67387676..fb501219672 100644 --- a/pkg/scheduler/factory.go +++ b/pkg/scheduler/factory.go @@ -31,6 +31,7 @@ import ( "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" corelisters "k8s.io/client-go/listers/core/v1" + restclient "k8s.io/client-go/rest" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -57,7 +58,8 @@ type Binder interface { // Configurator defines I/O, caching, and other functionality needed to // construct a new scheduler. type Configurator struct { - client clientset.Interface + client clientset.Interface + kubeConfig *restclient.Config recorderFactory profile.RecorderFactory @@ -137,6 +139,7 @@ func (c *Configurator) create() (*Scheduler, error) { clusterEventMap := make(map[framework.ClusterEvent]sets.String) profiles, err := profile.NewMap(c.profiles, c.registry, c.recorderFactory, frameworkruntime.WithClientSet(c.client), + frameworkruntime.WithKubeConfig(c.kubeConfig), frameworkruntime.WithInformerFactory(c.informerFactory), frameworkruntime.WithSnapshotSharedLister(c.nodeInfoSnapshot), frameworkruntime.WithRunAllFilters(c.alwaysCheckAllPredicates), diff --git a/pkg/scheduler/framework/interface.go b/pkg/scheduler/framework/interface.go index 87f5d8b8438..dedc053fab5 100644 --- a/pkg/scheduler/framework/interface.go +++ b/pkg/scheduler/framework/interface.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/internal/parallelize" @@ -588,6 +589,9 @@ type Handle interface { // ClientSet returns a kubernetes clientSet. ClientSet() clientset.Interface + // KubeConfig returns the raw kube config. + KubeConfig() *restclient.Config + // EventRecorder returns an event recorder. EventRecorder() events.EventRecorder diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 7177d2ba342..2fb83e0fd95 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" "k8s.io/component-helpers/scheduling/corev1" "k8s.io/klog/v2" @@ -92,6 +93,7 @@ type frameworkImpl struct { permitPlugins []framework.PermitPlugin clientSet clientset.Interface + kubeConfig *restclient.Config eventRecorder events.EventRecorder informerFactory informers.SharedInformerFactory @@ -142,6 +144,7 @@ func (f *frameworkImpl) Extenders() []framework.Extender { type frameworkOptions struct { clientSet clientset.Interface + kubeConfig *restclient.Config eventRecorder events.EventRecorder informerFactory informers.SharedInformerFactory snapshotSharedLister framework.SharedLister @@ -164,6 +167,13 @@ func WithClientSet(clientSet clientset.Interface) Option { } } +// WithKubeConfig sets kubeConfig for the scheduling frameworkImpl. +func WithKubeConfig(kubeConfig *restclient.Config) Option { + return func(o *frameworkOptions) { + o.kubeConfig = kubeConfig + } +} + // WithEventRecorder sets clientSet for the scheduling frameworkImpl. func WithEventRecorder(recorder events.EventRecorder) Option { return func(o *frameworkOptions) { @@ -254,6 +264,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti pluginNameToWeightMap: make(map[string]int), waitingPods: newWaitingPodsMap(), clientSet: options.clientSet, + kubeConfig: options.kubeConfig, eventRecorder: options.eventRecorder, informerFactory: options.informerFactory, metricsRecorder: options.metricsRecorder, @@ -1149,6 +1160,11 @@ func (f *frameworkImpl) ClientSet() clientset.Interface { return f.clientSet } +// KubeConfig returns a kubernetes config. +func (f *frameworkImpl) KubeConfig() *restclient.Config { + return f.kubeConfig +} + // EventRecorder returns an event recorder. func (f *frameworkImpl) EventRecorder() events.EventRecorder { return f.eventRecorder diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index e1de5812fd7..2454ad58043 100755 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -32,6 +32,7 @@ import ( "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" podutil "k8s.io/kubernetes/pkg/api/v1/pod" @@ -89,6 +90,7 @@ type Scheduler struct { } type schedulerOptions struct { + kubeConfig *restclient.Config schedulerAlgorithmSource schedulerapi.SchedulerAlgorithmSource percentageOfNodesToScore int32 podInitialBackoffSeconds int64 @@ -104,6 +106,13 @@ type schedulerOptions struct { // Option configures a Scheduler type Option func(*schedulerOptions) +// WithKubeConfig sets the kube config for Scheduler. +func WithKubeConfig(cfg *restclient.Config) Option { + return func(o *schedulerOptions) { + o.kubeConfig = cfg + } +} + // WithProfiles sets profiles for Scheduler. By default, there is one profile // with the name "default-scheduler". func WithProfiles(p ...schedulerapi.KubeSchedulerProfile) Option { @@ -214,6 +223,7 @@ func New(client clientset.Interface, configurator := &Configurator{ client: client, + kubeConfig: options.kubeConfig, recorderFactory: recorderFactory, informerFactory: informerFactory, schedulerCache: schedulerCache, diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index c9dfd7f8cb9..3126c301fbb 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -176,7 +176,8 @@ func TestSchedulerCreation(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - s, err := New(client, + s, err := New( + client, informerFactory, profile.NewRecorderFactory(eventBroadcaster), stopCh, @@ -456,7 +457,8 @@ func TestSchedulerMultipleProfilesScheduling(t *testing.T) { defer cancel() informerFactory := informers.NewSharedInformerFactory(client, 0) - sched, err := New(client, + sched, err := New( + client, informerFactory, profile.NewRecorderFactory(broadcaster), ctx.Done(), diff --git a/test/integration/scheduler/framework_test.go b/test/integration/scheduler/framework_test.go index bc86a329c36..b28cc7a1430 100644 --- a/test/integration/scheduler/framework_test.go +++ b/test/integration/scheduler/framework_test.go @@ -92,7 +92,7 @@ type BindPlugin struct { numBindCalled int PluginName string bindStatus *framework.Status - client *clientset.Clientset + client clientset.Interface pluginInvokeEventChan chan pluginInvokeEvent } diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 2e9cf69e178..37a2c00b59f 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -99,7 +99,7 @@ func mustSetupScheduler(config *config.KubeSchedulerConfiguration) (util.Shutdow // Not all config options will be effective but only those mostly related with scheduler performance will // be applied to start a scheduler, most of them are defined in `scheduler.schedulerOptions`. - _, podInformer, schedulerShutdown := util.StartScheduler(client, config) + _, podInformer, schedulerShutdown := util.StartScheduler(client, cfg, config) fakePVControllerShutdown := util.StartFakePVController(client) shutdownFunc := func() { diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 7967cb29e97..92581f50d5d 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -75,7 +75,7 @@ func StartApiserver() (string, ShutdownFunc) { // StartScheduler configures and starts a scheduler given a handle to the clientSet interface // and event broadcaster. It returns the running scheduler, podInformer and the shutdown function to stop it. -func StartScheduler(clientSet clientset.Interface, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) { +func StartScheduler(clientSet clientset.Interface, kubeConfig *restclient.Config, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) (*scheduler.Scheduler, coreinformers.PodInformer, ShutdownFunc) { ctx, cancel := context.WithCancel(context.Background()) informerFactory := scheduler.NewInformerFactory(clientSet, 0) @@ -89,6 +89,7 @@ func StartScheduler(clientSet clientset.Interface, cfg *kubeschedulerconfig.Kube informerFactory, profile.NewRecorderFactory(evtBroadcaster), ctx.Done(), + scheduler.WithKubeConfig(kubeConfig), scheduler.WithProfiles(cfg.Profiles...), scheduler.WithAlgorithmSource(cfg.AlgorithmSource), scheduler.WithPercentageOfNodesToScore(cfg.PercentageOfNodesToScore), @@ -159,7 +160,8 @@ type TestContext struct { CloseFn framework.CloseFunc HTTPServer *httptest.Server NS *v1.Namespace - ClientSet *clientset.Clientset + ClientSet clientset.Interface + KubeConfig *restclient.Config InformerFactory informers.SharedInformerFactory Scheduler *scheduler.Scheduler Ctx context.Context @@ -349,14 +351,14 @@ func InitTestMaster(t *testing.T, nsPrefix string, admission admission.Interface } // 2. Create kubeclient - testCtx.ClientSet = clientset.NewForConfigOrDie( - &restclient.Config{ - QPS: -1, Host: s.URL, - ContentConfig: restclient.ContentConfig{ - GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}, - }, + kubeConfig := &restclient.Config{ + QPS: -1, Host: s.URL, + ContentConfig: restclient.ContentConfig{ + GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}, }, - ) + } + testCtx.KubeConfig = kubeConfig + testCtx.ClientSet = clientset.NewForConfigOrDie(kubeConfig) return &testCtx } @@ -403,6 +405,7 @@ func InitTestSchedulerWithOptions( if policy != nil { opts = append(opts, scheduler.WithAlgorithmSource(CreateAlgorithmSourceFromPolicy(policy, testCtx.ClientSet))) } + opts = append(opts, scheduler.WithKubeConfig(testCtx.KubeConfig)) testCtx.Scheduler, err = scheduler.New( testCtx.ClientSet, testCtx.InformerFactory,