From 30e7016ccf92e0e2200023cfd3eb03b1ea8e5c18 Mon Sep 17 00:00:00 2001 From: Abdullah Gharaibeh Date: Wed, 2 Oct 2019 19:31:16 -0400 Subject: [PATCH] Refactor scheduler.New so that all framework-related parameters are passed as options --- cmd/kube-scheduler/app/server.go | 9 +- pkg/scheduler/BUILD | 1 + pkg/scheduler/api/compatibility/BUILD | 1 - .../api/compatibility/compatibility_test.go | 4 - pkg/scheduler/scheduler.go | 74 +++- pkg/scheduler/scheduler_test.go | 10 +- test/integration/daemonset/BUILD | 1 - test/integration/daemonset/daemonset_test.go | 4 - test/integration/scheduler/BUILD | 2 - test/integration/scheduler/framework_test.go | 333 +++++++----------- test/integration/scheduler/preemption_test.go | 17 +- test/integration/scheduler/scheduler_test.go | 9 +- test/integration/scheduler/util.go | 12 +- test/integration/util/BUILD | 1 - test/integration/util/util.go | 4 - test/integration/volumescheduling/BUILD | 2 - test/integration/volumescheduling/util.go | 11 +- 17 files changed, 216 insertions(+), 279 deletions(-) diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index cc16394c50f..e117feeee00 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -182,14 +182,15 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}, regis cc.Recorder, cc.ComponentConfig.AlgorithmSource, stopCh, - registry, - cc.ComponentConfig.Plugins, - cc.ComponentConfig.PluginConfig, scheduler.WithName(cc.ComponentConfig.SchedulerName), scheduler.WithHardPodAffinitySymmetricWeight(cc.ComponentConfig.HardPodAffinitySymmetricWeight), scheduler.WithPreemptionDisabled(cc.ComponentConfig.DisablePreemption), scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore), - scheduler.WithBindTimeoutSeconds(*cc.ComponentConfig.BindTimeoutSeconds)) + scheduler.WithBindTimeoutSeconds(*cc.ComponentConfig.BindTimeoutSeconds), + scheduler.WithFrameworkRegistry(registry), + scheduler.WithFrameworkPlugins(cc.ComponentConfig.Plugins), + scheduler.WithFrameworkPluginConfig(cc.ComponentConfig.PluginConfig), + ) if err != nil { return err } diff --git a/pkg/scheduler/BUILD b/pkg/scheduler/BUILD index f2eb66349e1..7c98fc2caec 100644 --- a/pkg/scheduler/BUILD +++ b/pkg/scheduler/BUILD @@ -15,6 +15,7 @@ go_library( "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/core:go_default_library", "//pkg/scheduler/factory:go_default_library", + "//pkg/scheduler/framework/plugins:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/internal/cache:go_default_library", "//pkg/scheduler/internal/queue:go_default_library", diff --git a/pkg/scheduler/api/compatibility/BUILD b/pkg/scheduler/api/compatibility/BUILD index 199c76388fb..3ea02c5ce7b 100644 --- a/pkg/scheduler/api/compatibility/BUILD +++ b/pkg/scheduler/api/compatibility/BUILD @@ -11,7 +11,6 @@ go_test( "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/core:go_default_library", "//pkg/scheduler/factory:go_default_library", - "//pkg/scheduler/framework/plugins:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/pkg/scheduler/api/compatibility/compatibility_test.go b/pkg/scheduler/api/compatibility/compatibility_test.go index 575d9635d71..09624889c8c 100644 --- a/pkg/scheduler/api/compatibility/compatibility_test.go +++ b/pkg/scheduler/api/compatibility/compatibility_test.go @@ -32,7 +32,6 @@ import ( schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/core" "k8s.io/kubernetes/pkg/scheduler/factory" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/plugins" ) func TestCompatibility_v1_Scheduler(t *testing.T) { @@ -1163,9 +1162,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { nil, algorithmSrc, make(chan struct{}), - schedulerframework.NewDefaultRegistry(), - nil, - []kubeschedulerconfig.PluginConfig{}, ) if err != nil { t.Fatalf("%s: Error constructing: %v", v, err) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index bc3d7f620dc..d9b949bf82c 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -40,6 +40,7 @@ import ( kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/core" "k8s.io/kubernetes/pkg/scheduler/factory" + frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" @@ -109,11 +110,15 @@ func (sched *Scheduler) Cache() internalcache.Cache { } type schedulerOptions struct { - schedulerName string - hardPodAffinitySymmetricWeight int32 - disablePreemption bool - percentageOfNodesToScore int32 - bindTimeoutSeconds int64 + schedulerName string + hardPodAffinitySymmetricWeight int32 + disablePreemption bool + percentageOfNodesToScore int32 + bindTimeoutSeconds int64 + frameworkRegistry framework.Registry + frameworkConfigProducerRegistry *frameworkplugins.ConfigProducerRegistry + frameworkPlugins *kubeschedulerconfig.Plugins + frameworkPluginConfig []kubeschedulerconfig.PluginConfig } // Option configures a Scheduler @@ -154,12 +159,51 @@ func WithBindTimeoutSeconds(bindTimeoutSeconds int64) Option { } } +// WithFrameworkRegistry sets the framework registry. +func WithFrameworkRegistry(registry framework.Registry) Option { + return func(o *schedulerOptions) { + o.frameworkRegistry = registry + } +} + +// WithFrameworkConfigProducerRegistry sets the framework plugin producer registry. +func WithFrameworkConfigProducerRegistry(registry *frameworkplugins.ConfigProducerRegistry) Option { + return func(o *schedulerOptions) { + o.frameworkConfigProducerRegistry = registry + } +} + +// WithFrameworkPlugins sets the plugins that the framework should be configured with. +func WithFrameworkPlugins(plugins *kubeschedulerconfig.Plugins) Option { + return func(o *schedulerOptions) { + o.frameworkPlugins = plugins + } +} + +// WithFrameworkPluginConfig sets the PluginConfig slice that the framework should be configured with. +func WithFrameworkPluginConfig(pluginConfig []kubeschedulerconfig.PluginConfig) Option { + return func(o *schedulerOptions) { + o.frameworkPluginConfig = pluginConfig + } +} + var defaultSchedulerOptions = schedulerOptions{ - schedulerName: v1.DefaultSchedulerName, - hardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, - disablePreemption: false, - percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, - bindTimeoutSeconds: BindTimeoutSeconds, + schedulerName: v1.DefaultSchedulerName, + hardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, + disablePreemption: false, + percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, + bindTimeoutSeconds: BindTimeoutSeconds, + frameworkRegistry: frameworkplugins.NewDefaultRegistry(), + frameworkConfigProducerRegistry: frameworkplugins.NewDefaultConfigProducerRegistry(), + // The plugins and pluginConfig options are currently nil because we currently don't have + // "default" plugins. All plugins that we run through the framework currently come from two + // sources: 1) specified in component config, in which case those two options should be + // set using their corresponding With* functions, 2) predicate/priority-mapped plugins, which + // pluginConfigProducerRegistry contains a mapping for and produces their configurations. + // TODO(ahg-g) Once predicates and priorities are migrated to natively run as plugins, the + // below two parameters will be populated accordingly. + frameworkPlugins: nil, + frameworkPluginConfig: nil, } // New returns a Scheduler @@ -178,9 +222,6 @@ func New(client clientset.Interface, recorder events.EventRecorder, schedulerAlgorithmSource kubeschedulerconfig.SchedulerAlgorithmSource, stopCh <-chan struct{}, - registry framework.Registry, - plugins *kubeschedulerconfig.Plugins, - pluginConfig []kubeschedulerconfig.PluginConfig, opts ...Option) (*Scheduler, error) { options := defaultSchedulerOptions @@ -205,9 +246,10 @@ func New(client clientset.Interface, DisablePreemption: options.disablePreemption, PercentageOfNodesToScore: options.percentageOfNodesToScore, BindTimeoutSeconds: options.bindTimeoutSeconds, - Registry: registry, - Plugins: plugins, - PluginConfig: pluginConfig, + Registry: options.frameworkRegistry, + PluginConfigProducerRegistry: options.frameworkConfigProducerRegistry, + Plugins: options.frameworkPlugins, + PluginConfig: options.frameworkPluginConfig, }) var config *factory.Config source := schedulerAlgorithmSource diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 9c39cea061a..a19104965b2 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -59,11 +59,9 @@ import ( var ( emptyPluginRegistry = framework.Registry{} - // emptyPluginConfig is an empty plugin config used in tests. - emptyPluginConfig []kubeschedulerconfig.PluginConfig // emptyFramework is an empty framework used in tests. // Note: If the test runs in goroutine, please don't use this variable to avoid a race condition. - emptyFramework, _ = framework.NewFramework(emptyPluginRegistry, nil, emptyPluginConfig) + emptyFramework, _ = framework.NewFramework(emptyPluginRegistry, nil, nil) ) type fakeBinder struct { @@ -178,7 +176,6 @@ func TestSchedulerCreation(t *testing.T) { testSource := "testProvider" eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1beta1().Events("")}) - defaultBindTimeout := int64(30) factory.RegisterFitPredicate("PredicateOne", PredicateOne) factory.RegisterPriorityFunction("PriorityOne", PriorityOne, 1) factory.RegisterAlgorithmProvider(testSource, sets.NewString("PredicateOne"), sets.NewString("PriorityOne")) @@ -200,10 +197,7 @@ func TestSchedulerCreation(t *testing.T) { eventBroadcaster.NewRecorder(scheme.Scheme, "scheduler"), kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &testSource}, stopCh, - emptyPluginRegistry, - nil, - emptyPluginConfig, - WithBindTimeoutSeconds(defaultBindTimeout)) + ) if err != nil { t.Fatalf("Failed to create scheduler: %v", err) diff --git a/test/integration/daemonset/BUILD b/test/integration/daemonset/BUILD index 21691753fca..2c289702393 100644 --- a/test/integration/daemonset/BUILD +++ b/test/integration/daemonset/BUILD @@ -23,7 +23,6 @@ go_test( "//pkg/scheduler/algorithmprovider:go_default_library", "//pkg/scheduler/api:go_default_library", "//pkg/scheduler/apis/config:go_default_library", - "//pkg/scheduler/framework/plugins:go_default_library", "//pkg/util/labels:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/test/integration/daemonset/daemonset_test.go b/test/integration/daemonset/daemonset_test.go index 6120ec8034d..dfe6a799f23 100644 --- a/test/integration/daemonset/daemonset_test.go +++ b/test/integration/daemonset/daemonset_test.go @@ -51,7 +51,6 @@ import ( "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/plugins" labelsutil "k8s.io/kubernetes/pkg/util/labels" "k8s.io/kubernetes/test/integration/framework" ) @@ -126,9 +125,6 @@ func setupScheduler( Provider: &defaultProviderName, }, stopCh, - schedulerframework.NewDefaultRegistry(), - nil, - []schedulerconfig.PluginConfig{}, ) if err != nil { t.Fatalf("Couldn't create scheduler: %v", err) diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index 057431bc709..8ff23ec5989 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -97,8 +97,6 @@ go_library( "//pkg/scheduler/api/latest:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/factory:go_default_library", - "//pkg/scheduler/framework/plugins:go_default_library", - "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/util/taints:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", diff --git a/test/integration/scheduler/framework_test.go b/test/integration/scheduler/framework_test.go index 19827d37e0b..049512f58c3 100644 --- a/test/integration/scheduler/framework_test.go +++ b/test/integration/scheduler/framework_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + scheduler "k8s.io/kubernetes/pkg/scheduler" schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" @@ -456,23 +457,13 @@ func TestPreFilterPlugin(t *testing.T) { }, }, } - // Set empty plugin config for testing - emptyPluginConfig := []schedulerconfig.PluginConfig{} // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "prefilter-plugin", nil), - false, nil, registry, plugins, emptyPluginConfig, time.Second) - + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "prefilter-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - tests := []struct { fail bool reject bool @@ -495,18 +486,18 @@ func TestPreFilterPlugin(t *testing.T) { preFilterPlugin.failPreFilter = test.fail preFilterPlugin.rejectPreFilter = test.reject // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if test.reject || test.fail { - if err = waitForPodUnschedulable(cs, pod); err != nil { + if err = waitForPodUnschedulable(context.clientSet, pod); err != nil { t.Errorf("test #%v: Didn't expect the pod to be scheduled. error: %v", i, err) } } else { - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err) } } @@ -516,7 +507,7 @@ func TestPreFilterPlugin(t *testing.T) { } preFilterPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -538,27 +529,30 @@ func TestScorePlugin(t *testing.T) { }, }, } - context, cs := initTestContextForScorePlugin(t, plugins, registry) + + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "score-plugin", nil), 10, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) for i, fail := range []bool{false, true} { scorePlugin.failScore = fail // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Fatalf("Error while creating a test pod: %v", err) } if fail { - if err = waitForPodUnschedulable(cs, pod); err != nil { + if err = waitForPodUnschedulable(context.clientSet, pod); err != nil { t.Errorf("test #%v: Didn't expect the pod to be scheduled. error: %v", i, err) } } else { - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("Expected the pod to be scheduled. error: %v", err) } else { - p, err := getPod(cs, pod.Name, pod.Namespace) + p, err := getPod(context.clientSet, pod.Name, pod.Namespace) if err != nil { t.Errorf("Failed to retrieve the pod. error: %v", err) } else if p.Spec.NodeName != scorePlugin.highScoreNode { @@ -572,7 +566,7 @@ func TestScorePlugin(t *testing.T) { } scorePlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -594,17 +588,20 @@ func TestNormalizeScorePlugin(t *testing.T) { }, }, } - context, cs := initTestContextForScorePlugin(t, plugins, registry) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "score-plugin", nil), 10, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkRegistry(registry)) + defer cleanupTest(t, context) // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Fatalf("Error while creating a test pod: %v", err) } - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("Expected the pod to be scheduled. error: %v", err) } @@ -634,37 +631,29 @@ func TestReservePlugin(t *testing.T) { }, }, } - // Set empty plugin config for testing - emptyPluginConfig := []schedulerconfig.PluginConfig{} // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "reserve-plugin", nil), - false, nil, registry, plugins, emptyPluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "reserve-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - for _, fail := range []bool{false, true} { reservePlugin.failReserve = fail // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if fail { - if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(cs, pod.Namespace, pod.Name)); err != nil { + if err = wait.Poll(10*time.Millisecond, 30*time.Second, + podSchedulingError(context.clientSet, pod.Namespace, pod.Name)); err != nil { t.Errorf("Didn't expect the pod to be scheduled. error: %v", err) } } else { - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("Expected the pod to be scheduled. error: %v", err) } } @@ -674,7 +663,7 @@ func TestReservePlugin(t *testing.T) { } reservePlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -702,18 +691,12 @@ func TestPrebindPlugin(t *testing.T) { }, } // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "prebind-plugin", nil), - false, nil, registry, plugins, preBindPluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "prebind-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkPluginConfig(preBindPluginConfig), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - tests := []struct { fail bool reject bool @@ -740,17 +723,17 @@ func TestPrebindPlugin(t *testing.T) { preBindPlugin.failPreBind = test.fail preBindPlugin.rejectPreBind = test.reject // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if test.fail || test.reject { - if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(cs, pod.Namespace, pod.Name)); err != nil { + if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(context.clientSet, pod.Namespace, pod.Name)); err != nil { t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err) } - } else if err = waitForPodToSchedule(cs, pod); err != nil { + } else if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err) } @@ -759,7 +742,7 @@ func TestPrebindPlugin(t *testing.T) { } preBindPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -803,18 +786,12 @@ func TestUnreservePlugin(t *testing.T) { } // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "unreserve-plugin", nil), - false, nil, registry, plugins, pluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "unreserve-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkPluginConfig(pluginConfig), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - tests := []struct { preBindFail bool }{ @@ -830,21 +807,21 @@ func TestUnreservePlugin(t *testing.T) { preBindPlugin.failPreBind = test.preBindFail // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if test.preBindFail { - if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(cs, pod.Namespace, pod.Name)); err != nil { + if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(context.clientSet, pod.Namespace, pod.Name)); err != nil { t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err) } if unreservePlugin.numUnreserveCalled == 0 || unreservePlugin.numUnreserveCalled != preBindPlugin.numPreBindCalled { t.Errorf("test #%v: Expected the unreserve plugin to be called %d times, was called %d times.", i, preBindPlugin.numPreBindCalled, unreservePlugin.numUnreserveCalled) } } else { - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err) } if unreservePlugin.numUnreserveCalled > 0 { @@ -854,7 +831,7 @@ func TestUnreservePlugin(t *testing.T) { unreservePlugin.reset() preBindPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -919,13 +896,14 @@ func TestBindPlugin(t *testing.T) { } // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, testContext, - false, nil, registry, plugins, pluginConfig, time.Second) + context := initTestSchedulerWithOptions(t, testContext, false, nil, time.Second, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkPluginConfig(pluginConfig), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) + _, err := createNodes(context.clientSet, "test-node", nil, 2) if err != nil { t.Fatalf("Cannot create nodes: %v", err) } @@ -976,19 +954,19 @@ func TestBindPlugin(t *testing.T) { postBindPlugin.pluginInvokeEventChan = pluginInvokeEventChan // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if test.expectBoundByScheduler || test.expectBoundByPlugin { // bind plugins skipped to bind the pod - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err) continue } - pod, err = cs.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) + pod, err = context.clientSet.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) if err != nil { t.Errorf("can't get pod: %v", err) } @@ -1021,7 +999,7 @@ func TestBindPlugin(t *testing.T) { } } else { // bind plugin fails to bind the pod - if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(cs, pod.Namespace, pod.Name)); err != nil { + if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(context.clientSet, pod.Namespace, pod.Name)); err != nil { t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err) } if postBindPlugin.numPostBindCalled > 0 { @@ -1046,7 +1024,7 @@ func TestBindPlugin(t *testing.T) { bindPlugin1.reset() bindPlugin2.reset() unreservePlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -1090,18 +1068,12 @@ func TestPostBindPlugin(t *testing.T) { } // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "postbind-plugin", nil), - false, nil, registry, plugins, pluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "postbind-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkPluginConfig(pluginConfig), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - tests := []struct { preBindFail bool preBindReject bool @@ -1118,21 +1090,21 @@ func TestPostBindPlugin(t *testing.T) { preBindPlugin.failPreBind = test.preBindFail // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if test.preBindFail { - if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(cs, pod.Namespace, pod.Name)); err != nil { + if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(context.clientSet, pod.Namespace, pod.Name)); err != nil { t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err) } if postBindPlugin.numPostBindCalled > 0 { t.Errorf("test #%v: Didn't expected the postbind plugin to be called %d times.", i, postBindPlugin.numPostBindCalled) } } else { - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err) } if postBindPlugin.numPostBindCalled == 0 { @@ -1142,7 +1114,7 @@ func TestPostBindPlugin(t *testing.T) { postBindPlugin.reset() preBindPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -1171,18 +1143,12 @@ func TestPermitPlugin(t *testing.T) { } // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "permit-plugin", nil), - false, nil, registry, plugins, pluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "permit-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkPluginConfig(pluginConfig), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - tests := []struct { fail bool reject bool @@ -1228,22 +1194,22 @@ func TestPermitPlugin(t *testing.T) { perPlugin.waitAndAllowPermit = false // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if test.fail { - if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(cs, pod.Namespace, pod.Name)); err != nil { + if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(context.clientSet, pod.Namespace, pod.Name)); err != nil { t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err) } } else { if test.reject || test.timeout { - if err = waitForPodUnschedulable(cs, pod); err != nil { + if err = waitForPodUnschedulable(context.clientSet, pod); err != nil { t.Errorf("test #%v: Didn't expect the pod to be scheduled. error: %v", i, err) } } else { - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err) } } @@ -1254,7 +1220,7 @@ func TestPermitPlugin(t *testing.T) { } perPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -1283,18 +1249,12 @@ func TestCoSchedulingWithPermitPlugin(t *testing.T) { } // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "permit-plugin", nil), - false, nil, registry, plugins, pluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "permit-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkPluginConfig(pluginConfig), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - tests := []struct { waitReject bool waitAllow bool @@ -1317,29 +1277,29 @@ func TestCoSchedulingWithPermitPlugin(t *testing.T) { permitPlugin.waitAndAllowPermit = test.waitAllow // Create two pods. - waitingPod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "waiting-pod", Namespace: context.ns.Name})) + waitingPod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "waiting-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating the waiting pod: %v", err) } - signallingPod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "signalling-pod", Namespace: context.ns.Name})) + signallingPod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "signalling-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating the signalling pod: %v", err) } if test.waitReject { - if err = waitForPodUnschedulable(cs, waitingPod); err != nil { + if err = waitForPodUnschedulable(context.clientSet, waitingPod); err != nil { t.Errorf("test #%v: Didn't expect the waiting pod to be scheduled. error: %v", i, err) } - if err = waitForPodUnschedulable(cs, signallingPod); err != nil { + if err = waitForPodUnschedulable(context.clientSet, signallingPod); err != nil { t.Errorf("test #%v: Didn't expect the signalling pod to be scheduled. error: %v", i, err) } } else { - if err = waitForPodToSchedule(cs, waitingPod); err != nil { + if err = waitForPodToSchedule(context.clientSet, waitingPod); err != nil { t.Errorf("test #%v: Expected the waiting pod to be scheduled. error: %v", i, err) } - if err = waitForPodToSchedule(cs, signallingPod); err != nil { + if err = waitForPodToSchedule(context.clientSet, signallingPod); err != nil { t.Errorf("test #%v: Expected the signalling pod to be scheduled. error: %v", i, err) } } @@ -1349,7 +1309,7 @@ func TestCoSchedulingWithPermitPlugin(t *testing.T) { } permitPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{waitingPod, signallingPod}) + cleanupPods(context.clientSet, t, []*v1.Pod{waitingPod, signallingPod}) } } @@ -1360,7 +1320,7 @@ func TestFilterPlugin(t *testing.T) { registry := framework.Registry{filterPluginName: newPlugin(filterPlugin)} // Setup initial filter plugin for testing. - plugin := &schedulerconfig.Plugins{ + plugins := &schedulerconfig.Plugins{ Filter: &schedulerconfig.PluginSet{ Enabled: []schedulerconfig.Plugin{ { @@ -1369,37 +1329,28 @@ func TestFilterPlugin(t *testing.T) { }, }, } - // Set empty plugin config for testing - emptyPluginConfig := []schedulerconfig.PluginConfig{} // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "filter-plugin", nil), - false, nil, registry, plugin, emptyPluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "filter-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - for _, fail := range []bool{false, true} { filterPlugin.failFilter = fail // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if fail { - if err = wait.Poll(10*time.Millisecond, 30*time.Second, podUnschedulable(cs, pod.Namespace, pod.Name)); err != nil { + if err = wait.Poll(10*time.Millisecond, 30*time.Second, podUnschedulable(context.clientSet, pod.Namespace, pod.Name)); err != nil { t.Errorf("Didn't expect the pod to be scheduled.") } } else { - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("Expected the pod to be scheduled. error: %v", err) } } @@ -1409,7 +1360,7 @@ func TestFilterPlugin(t *testing.T) { } filterPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -1420,7 +1371,7 @@ func TestPostFilterPlugin(t *testing.T) { registry := framework.Registry{postFilterPluginName: newPlugin(postFilterPlugin)} // Setup initial post-filter plugin for testing. - pluginsConfig := &schedulerconfig.Plugins{ + plugins := &schedulerconfig.Plugins{ PostFilter: &schedulerconfig.PluginSet{ Enabled: []schedulerconfig.Plugin{ { @@ -1429,37 +1380,28 @@ func TestPostFilterPlugin(t *testing.T) { }, }, } - // Set empty plugin config for testing - emptyPluginConfig := []schedulerconfig.PluginConfig{} // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "post-filter-plugin", nil), - false, nil, registry, pluginsConfig, emptyPluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "post-filter-plugin", nil), 2, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet - // Add a few nodes. - _, err := createNodes(cs, "test-node", nil, 2) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) - } - for _, fail := range []bool{false, true} { postFilterPlugin.failPostFilter = fail // Create a best effort pod. - pod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) + pod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "test-pod", Namespace: context.ns.Name})) if err != nil { t.Errorf("Error while creating a test pod: %v", err) } if fail { - if err = waitForPodUnschedulable(cs, pod); err != nil { + if err = waitForPodUnschedulable(context.clientSet, pod); err != nil { t.Errorf("Didn't expect the pod to be scheduled. error: %v", err) } } else { - if err = waitForPodToSchedule(cs, pod); err != nil { + if err = waitForPodToSchedule(context.clientSet, pod); err != nil { t.Errorf("Expected the pod to be scheduled. error: %v", err) } } @@ -1469,7 +1411,7 @@ func TestPostFilterPlugin(t *testing.T) { } postFilterPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{pod}) + cleanupPods(context.clientSet, t, []*v1.Pod{pod}) } } @@ -1498,19 +1440,19 @@ func TestPreemptWithPermitPlugin(t *testing.T) { } // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "preempt-with-permit-plugin", nil), - false, nil, registry, plugins, pluginConfig, time.Second) + context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "preempt-with-permit-plugin", nil), 0, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkPluginConfig(pluginConfig), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) - cs := context.clientSet // Add one node. nodeRes := &v1.ResourceList{ v1.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), v1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(500, resource.DecimalSI), } - _, err := createNodes(cs, "test-node", nodeRes, 1) + _, err := createNodes(context.clientSet, "test-node", nodeRes, 1) if err != nil { t.Fatalf("Cannot create nodes: %v", err) } @@ -1529,9 +1471,9 @@ func TestPreemptWithPermitPlugin(t *testing.T) { } // Create two pods. - waitingPod := initPausePod(cs, &pausePodConfig{Name: "waiting-pod", Namespace: context.ns.Name, Priority: &lowPriority, Resources: &resourceRequest}) + waitingPod := initPausePod(context.clientSet, &pausePodConfig{Name: "waiting-pod", Namespace: context.ns.Name, Priority: &lowPriority, Resources: &resourceRequest}) waitingPod.Spec.TerminationGracePeriodSeconds = new(int64) - waitingPod, err = createPausePod(cs, waitingPod) + waitingPod, err = createPausePod(context.clientSet, waitingPod) if err != nil { t.Errorf("Error while creating the waiting pod: %v", err) } @@ -1542,17 +1484,17 @@ func TestPreemptWithPermitPlugin(t *testing.T) { return w, nil }) - preemptorPod, err := createPausePod(cs, - initPausePod(cs, &pausePodConfig{Name: "preemptor-pod", Namespace: context.ns.Name, Priority: &highPriority, Resources: &resourceRequest})) + preemptorPod, err := createPausePod(context.clientSet, + initPausePod(context.clientSet, &pausePodConfig{Name: "preemptor-pod", Namespace: context.ns.Name, Priority: &highPriority, Resources: &resourceRequest})) if err != nil { t.Errorf("Error while creating the preemptor pod: %v", err) } - if err = waitForPodToSchedule(cs, preemptorPod); err != nil { + if err = waitForPodToSchedule(context.clientSet, preemptorPod); err != nil { t.Errorf("Expected the preemptor pod to be scheduled. error: %v", err) } - if _, err := getPod(cs, waitingPod.Name, waitingPod.Namespace); err == nil { + if _, err := getPod(context.clientSet, waitingPod.Name, waitingPod.Namespace); err == nil { t.Error("Expected the waiting pod to get preempted and deleted") } @@ -1561,22 +1503,17 @@ func TestPreemptWithPermitPlugin(t *testing.T) { } permitPlugin.reset() - cleanupPods(cs, t, []*v1.Pod{waitingPod, preemptorPod}) + cleanupPods(context.clientSet, t, []*v1.Pod{waitingPod, preemptorPod}) } -func initTestContextForScorePlugin(t *testing.T, plugins *schedulerconfig.Plugins, registry framework.Registry) (*testContext, *clientset.Clientset) { - // Set empty plugin config for testing - emptyPluginConfig := []schedulerconfig.PluginConfig{} - - // Create the master and the scheduler with the test plugin set. - context := initTestSchedulerWithOptions(t, - initTestMaster(t, "score-plugin", nil), - false, nil, registry, plugins, emptyPluginConfig, time.Second) - - cs := context.clientSet - _, err := createNodes(cs, "test-node", nil, 10) - if err != nil { - t.Fatalf("Cannot create nodes: %v", err) +func initTestSchedulerForFrameworkTest(t *testing.T, context *testContext, nodeCount int, opts ...scheduler.Option) *testContext { + opts = append(opts, scheduler.WithFrameworkConfigProducerRegistry(nil)) + c := initTestSchedulerWithOptions(t, context, false, nil, time.Second, opts...) + if nodeCount > 0 { + _, err := createNodes(c.clientSet, "test-node", nil, nodeCount) + if err != nil { + t.Fatalf("Cannot create nodes: %v", err) + } } - return context, cs + return c } diff --git a/test/integration/scheduler/preemption_test.go b/test/integration/scheduler/preemption_test.go index b9ee94c85ab..bcbebed6c16 100644 --- a/test/integration/scheduler/preemption_test.go +++ b/test/integration/scheduler/preemption_test.go @@ -38,13 +38,16 @@ import ( restclient "k8s.io/client-go/rest" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/apis/scheduling" - _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" + "k8s.io/kubernetes/pkg/scheduler" schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" + frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" "k8s.io/kubernetes/plugin/pkg/admission/priority" testutils "k8s.io/kubernetes/test/utils" + _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" + "k8s.io/klog" ) @@ -121,10 +124,12 @@ var _ = framework.FilterPlugin(&tokenFilter{}) func TestPreemption(t *testing.T) { // Initialize scheduler with a filter plugin. var filter tokenFilter - registry := framework.Registry{filterPluginName: func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { + registry := frameworkplugins.NewDefaultRegistry() + + registry.Register(filterPluginName, func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { return &filter, nil - }} - plugin := &schedulerconfig.Plugins{ + }) + plugins := &schedulerconfig.Plugins{ Filter: &schedulerconfig.PluginSet{ Enabled: []schedulerconfig.Plugin{ { @@ -142,7 +147,9 @@ func TestPreemption(t *testing.T) { } context := initTestSchedulerWithOptions(t, initTestMaster(t, "preemptiom", nil), - false, nil, registry, plugin, []schedulerconfig.PluginConfig{}, time.Second) + false, nil, time.Second, + scheduler.WithFrameworkPlugins(plugins), + scheduler.WithFrameworkRegistry(registry)) defer cleanupTest(t, context) cs := context.clientSet diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index e8fb130f375..2659178c37c 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -41,7 +41,6 @@ import ( _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/factory" - schedulerplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" "k8s.io/kubernetes/test/integration/framework" @@ -265,9 +264,6 @@ priorities: [] }, }, nil, - schedulerplugins.NewDefaultRegistry(), - nil, - []kubeschedulerconfig.PluginConfig{}, scheduler.WithName(v1.DefaultSchedulerName), scheduler.WithHardPodAffinitySymmetricWeight(v1.DefaultHardPodAffinitySymmetricWeight), scheduler.WithBindTimeoutSeconds(defaultBindTimeout), @@ -336,9 +332,6 @@ func TestSchedulerCreationFromNonExistentConfigMap(t *testing.T) { }, }, nil, - schedulerplugins.NewDefaultRegistry(), - nil, - []kubeschedulerconfig.PluginConfig{}, scheduler.WithName(v1.DefaultSchedulerName), scheduler.WithHardPodAffinitySymmetricWeight(v1.DefaultHardPodAffinitySymmetricWeight), scheduler.WithBindTimeoutSeconds(defaultBindTimeout)) @@ -596,7 +589,7 @@ func TestMultiScheduler(t *testing.T) { } // 5. create and start a scheduler with name "foo-scheduler" - context = initTestSchedulerWithOptions(t, context, true, nil, schedulerplugins.NewDefaultRegistry(), nil, []kubeschedulerconfig.PluginConfig{}, time.Second, scheduler.WithName(fooScheduler)) + context = initTestSchedulerWithOptions(t, context, true, nil, time.Second, scheduler.WithName(fooScheduler)) // 6. **check point-2**: // - testPodWithAnnotationFitsFoo should be scheduled diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index 9e828c26e13..5463105f654 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -56,8 +56,6 @@ import ( _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "k8s.io/kubernetes/pkg/scheduler/factory" - schedulerplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" taintutils "k8s.io/kubernetes/pkg/util/taints" "k8s.io/kubernetes/test/integration/framework" imageutils "k8s.io/kubernetes/test/utils/image" @@ -142,8 +140,7 @@ func initTestScheduler( policy *schedulerapi.Policy, ) *testContext { // Pod preemption is enabled by default scheduler configuration. - return initTestSchedulerWithOptions(t, context, setPodInformer, policy, schedulerplugins.NewDefaultRegistry(), - nil, []schedulerconfig.PluginConfig{}, time.Second) + return initTestSchedulerWithOptions(t, context, setPodInformer, policy, time.Second) } // initTestSchedulerWithOptions initializes a test environment and creates a scheduler with default @@ -153,9 +150,6 @@ func initTestSchedulerWithOptions( context *testContext, setPodInformer bool, policy *schedulerapi.Policy, - pluginRegistry schedulerframework.Registry, - plugins *schedulerconfig.Plugins, - pluginConfig []schedulerconfig.PluginConfig, resyncPeriod time.Duration, opts ...scheduler.Option, ) *testContext { @@ -204,9 +198,6 @@ func initTestSchedulerWithOptions( recorder, algorithmSrc, context.stopCh, - pluginRegistry, - plugins, - pluginConfig, opts..., ) @@ -273,7 +264,6 @@ func initTest(t *testing.T, nsPrefix string) *testContext { func initTestDisablePreemption(t *testing.T, nsPrefix string) *testContext { return initTestSchedulerWithOptions( t, initTestMaster(t, nsPrefix, nil), true, nil, - schedulerplugins.NewDefaultRegistry(), nil, []schedulerconfig.PluginConfig{}, time.Second, scheduler.WithPreemptionDisabled(true)) } diff --git a/test/integration/util/BUILD b/test/integration/util/BUILD index 4a55bef4625..96c8d1297d7 100644 --- a/test/integration/util/BUILD +++ b/test/integration/util/BUILD @@ -17,7 +17,6 @@ go_library( "//pkg/scheduler:go_default_library", "//pkg/scheduler/algorithmprovider/defaults:go_default_library", "//pkg/scheduler/apis/config:go_default_library", - "//pkg/scheduler/framework/plugins:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 5f6e78061e8..3ccafe0f205 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -31,7 +31,6 @@ import ( // import DefaultProvider _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider/defaults" schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - schedulerplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" "k8s.io/kubernetes/test/integration/framework" ) @@ -124,8 +123,5 @@ func createScheduler( Provider: &defaultProviderName, }, stopCh, - schedulerplugins.NewDefaultRegistry(), - nil, - []schedulerconfig.PluginConfig{}, ) } diff --git a/test/integration/volumescheduling/BUILD b/test/integration/volumescheduling/BUILD index 072a2bdc1d5..ce7ddf6405c 100644 --- a/test/integration/volumescheduling/BUILD +++ b/test/integration/volumescheduling/BUILD @@ -60,8 +60,6 @@ go_library( "//pkg/scheduler:go_default_library", "//pkg/scheduler/algorithmprovider/defaults:go_default_library", "//pkg/scheduler/apis/config:go_default_library", - "//pkg/scheduler/framework/plugins:go_default_library", - "//pkg/scheduler/framework/v1alpha1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/test/integration/volumescheduling/util.go b/test/integration/volumescheduling/util.go index a7333817e5a..9598425536f 100644 --- a/test/integration/volumescheduling/util.go +++ b/test/integration/volumescheduling/util.go @@ -37,8 +37,6 @@ import ( podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/scheduler" schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - schedulerplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/test/integration/framework" // Install "DefaultProvider" algorithprovider @@ -116,8 +114,7 @@ func initTestSchedulerWithOptions( var err error context.scheduler, err = createSchedulerWithPodInformer( - context.clientSet, podInformer, context.informerFactory, schedulerplugins.NewDefaultRegistry(), nil, - []schedulerconfig.PluginConfig{}, recorder, context.stopCh) + context.clientSet, podInformer, context.informerFactory, recorder, context.stopCh) if err != nil { t.Fatalf("Couldn't create scheduler: %v", err) @@ -138,9 +135,6 @@ func createSchedulerWithPodInformer( clientSet clientset.Interface, podInformer coreinformers.PodInformer, informerFactory informers.SharedInformerFactory, - pluginRegistry schedulerframework.Registry, - plugins *schedulerconfig.Plugins, - pluginConfig []schedulerconfig.PluginConfig, recorder events.EventRecorder, stopCh <-chan struct{}, ) (*scheduler.Scheduler, error) { @@ -164,9 +158,6 @@ func createSchedulerWithPodInformer( Provider: &defaultProviderName, }, stopCh, - pluginRegistry, - plugins, - pluginConfig, ) }