From 65e9826f79321edc49e923be8b5c6583ebea26f6 Mon Sep 17 00:00:00 2001 From: Jan Chaloupka Date: Wed, 13 May 2020 11:39:56 +0200 Subject: [PATCH] integration: start informer and scheduler outside of InitTestScheduler In case two or more controllers share the informers created through InitTestScheduler, it's not safe to start the informers until all controllers set their informer indexers. Otherwise, some controller might fail to register their indexers in time. Thus, it's responsibility of each consumer to make sure all informers are started after all controllers had time to get initiliazed. --- test/integration/node/lifecycle_test.go | 11 ++++++----- test/integration/scheduler/extender_test.go | 2 ++ test/integration/scheduler/framework_test.go | 13 +++++++++---- test/integration/scheduler/preemption_test.go | 8 ++++++++ test/integration/scheduler/scheduler_test.go | 2 ++ test/integration/scheduler/taint_test.go | 10 ++++++---- test/integration/scheduler/util.go | 10 ++++++++-- test/integration/util/util.go | 11 ++++++----- 8 files changed, 47 insertions(+), 20 deletions(-) diff --git a/test/integration/node/lifecycle_test.go b/test/integration/node/lifecycle_test.go index 92732a9e5d6..a8f5e58eca4 100644 --- a/test/integration/node/lifecycle_test.go +++ b/test/integration/node/lifecycle_test.go @@ -161,13 +161,14 @@ func TestTaintBasedEvictions(t *testing.T) { return } - go nc.Run(testCtx.Ctx.Done()) - - // Waiting for all controller sync. + // Waiting for all controllers to sync externalInformers.Start(testCtx.Ctx.Done()) externalInformers.WaitForCacheSync(testCtx.Ctx.Done()) - testCtx.InformerFactory.Start(testCtx.Ctx.Done()) - testCtx.InformerFactory.WaitForCacheSync(testCtx.Ctx.Done()) + testutils.SyncInformerFactory(testCtx) + + // Run all controllers + go nc.Run(testCtx.Ctx.Done()) + go testCtx.Scheduler.Run(testCtx.Ctx) nodeRes := v1.ResourceList{ v1.ResourceCPU: resource.MustParse("4000m"), diff --git a/test/integration/scheduler/extender_test.go b/test/integration/scheduler/extender_test.go index b3de3e06262..47ed7bf349d 100644 --- a/test/integration/scheduler/extender_test.go +++ b/test/integration/scheduler/extender_test.go @@ -351,6 +351,8 @@ func TestSchedulerExtender(t *testing.T) { policy.APIVersion = "v1" testCtx = testutils.InitTestScheduler(t, testCtx, false, &policy) + testutils.SyncInformerFactory(testCtx) + go testCtx.Scheduler.Run(testCtx.Ctx) defer testutils.CleanupTest(t, testCtx) DoTestPodScheduling(testCtx.NS, t, clientSet) diff --git a/test/integration/scheduler/framework_test.go b/test/integration/scheduler/framework_test.go index cf54a938e87..dffac82fe3e 100644 --- a/test/integration/scheduler/framework_test.go +++ b/test/integration/scheduler/framework_test.go @@ -892,10 +892,12 @@ func TestBindPlugin(t *testing.T) { }, } - // Create the master and the scheduler with the test plugin set. + // Create the scheduler with the test plugin set. testCtx := testutils.InitTestSchedulerWithOptions(t, testContext, false, nil, time.Second, scheduler.WithProfiles(prof), scheduler.WithFrameworkOutOfTreeRegistry(registry)) + testutils.SyncInformerFactory(testCtx) + go testCtx.Scheduler.Run(testCtx.Ctx) defer testutils.CleanupTest(t, testCtx) // Add a few nodes. @@ -1552,14 +1554,17 @@ func TestPreemptWithPermitPlugin(t *testing.T) { } func initTestSchedulerForFrameworkTest(t *testing.T, testCtx *testutils.TestContext, nodeCount int, opts ...scheduler.Option) *testutils.TestContext { - c := testutils.InitTestSchedulerWithOptions(t, testCtx, false, nil, time.Second, opts...) + testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, false, nil, time.Second, opts...) + testutils.SyncInformerFactory(testCtx) + go testCtx.Scheduler.Run(testCtx.Ctx) + if nodeCount > 0 { - _, err := createNodes(c.ClientSet, "test-node", nil, nodeCount) + _, err := createNodes(testCtx.ClientSet, "test-node", nil, nodeCount) if err != nil { t.Fatalf("Cannot create nodes: %v", err) } } - return c + return testCtx } // initRegistryAndConfig returns registry and plugins config based on give plugins. diff --git a/test/integration/scheduler/preemption_test.go b/test/integration/scheduler/preemption_test.go index 0f3d4ed9406..47cf213b73a 100644 --- a/test/integration/scheduler/preemption_test.go +++ b/test/integration/scheduler/preemption_test.go @@ -148,6 +148,8 @@ func TestPreemption(t *testing.T) { false, nil, time.Second, scheduler.WithProfiles(prof), scheduler.WithFrameworkOutOfTreeRegistry(registry)) + testutils.SyncInformerFactory(testCtx) + go testCtx.Scheduler.Run(testCtx.Ctx) defer testutils.CleanupTest(t, testCtx) cs := testCtx.ClientSet @@ -527,9 +529,15 @@ func TestPodPriorityResolution(t *testing.T) { externalInformers := informers.NewSharedInformerFactory(externalClientset, time.Second) admission.SetExternalKubeClientSet(externalClientset) admission.SetExternalKubeInformerFactory(externalInformers) + + // Waiting for all controllers to sync + testutils.SyncInformerFactory(testCtx) externalInformers.Start(testCtx.Ctx.Done()) externalInformers.WaitForCacheSync(testCtx.Ctx.Done()) + // Run all controllers + go testCtx.Scheduler.Run(testCtx.Ctx) + tests := []struct { Name string PriorityClass string diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 882f3b232dd..e5b9e087227 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -539,6 +539,8 @@ func TestMultipleSchedulers(t *testing.T) { // 5. create and start a scheduler with name "foo-scheduler" fooProf := kubeschedulerconfig.KubeSchedulerProfile{SchedulerName: fooScheduler} testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, true, nil, time.Second, scheduler.WithProfiles(fooProf)) + testutils.SyncInformerFactory(testCtx) + go testCtx.Scheduler.Run(testCtx.Ctx) // 6. **check point-2**: // - testPodWithAnnotationFitsFoo should be scheduled diff --git a/test/integration/scheduler/taint_test.go b/test/integration/scheduler/taint_test.go index 14bd73dbf19..bd7c3e91b5c 100644 --- a/test/integration/scheduler/taint_test.go +++ b/test/integration/scheduler/taint_test.go @@ -102,13 +102,15 @@ func TestTaintNodeByCondition(t *testing.T) { t.Errorf("Failed to create node controller: %v", err) return } - go nc.Run(testCtx.Ctx.Done()) - // Waiting for all controller sync. + // Waiting for all controllers to sync externalInformers.Start(testCtx.Ctx.Done()) externalInformers.WaitForCacheSync(testCtx.Ctx.Done()) - testCtx.InformerFactory.Start(testCtx.Ctx.Done()) - testCtx.InformerFactory.WaitForCacheSync(testCtx.Ctx.Done()) + testutils.SyncInformerFactory(testCtx) + + // Run all controllers + go nc.Run(testCtx.Ctx.Done()) + go testCtx.Scheduler.Run(testCtx.Ctx) // ------------------------------------------- // Test TaintNodeByCondition feature. diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index 6ab2c515514..ed0487d2140 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -79,15 +79,21 @@ func initDisruptionController(t *testing.T, testCtx *testutils.TestContext) *dis // initTest initializes a test environment and creates master and scheduler with default // configuration. func initTest(t *testing.T, nsPrefix string, opts ...scheduler.Option) *testutils.TestContext { - return testutils.InitTestSchedulerWithOptions(t, testutils.InitTestMaster(t, nsPrefix, nil), true, nil, time.Second, opts...) + testCtx := testutils.InitTestSchedulerWithOptions(t, testutils.InitTestMaster(t, nsPrefix, nil), true, nil, time.Second, opts...) + testutils.SyncInformerFactory(testCtx) + go testCtx.Scheduler.Run(testCtx.Ctx) + return testCtx } // initTestDisablePreemption initializes a test environment and creates master and scheduler with default // configuration but with pod preemption disabled. func initTestDisablePreemption(t *testing.T, nsPrefix string) *testutils.TestContext { - return testutils.InitTestSchedulerWithOptions( + testCtx := testutils.InitTestSchedulerWithOptions( t, testutils.InitTestMaster(t, nsPrefix, nil), true, nil, time.Second, scheduler.WithPreemptionDisabled(true)) + testutils.SyncInformerFactory(testCtx) + go testCtx.Scheduler.Run(testCtx.Ctx) + return testCtx } // waitForReflection waits till the passFunc confirms that the object it expects diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 4e740f9e40b..c92afeb1600 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -181,6 +181,12 @@ func PodDeleted(c clientset.Interface, podNamespace, podName string) wait.Condit } } +// SyncInformerFactory starts informer and waits for caches to be synced +func SyncInformerFactory(testCtx *TestContext) { + testCtx.InformerFactory.Start(testCtx.Ctx.Done()) + testCtx.InformerFactory.WaitForCacheSync(testCtx.Ctx.Done()) +} + // CleanupTest cleans related resources which were created during integration test func CleanupTest(t *testing.T, testCtx *TestContext) { // Kill the scheduler. @@ -408,11 +414,6 @@ func InitTestSchedulerWithOptions( stopCh := make(chan struct{}) eventBroadcaster.StartRecordingToSink(stopCh) - testCtx.InformerFactory.Start(testCtx.Scheduler.StopEverything) - testCtx.InformerFactory.WaitForCacheSync(testCtx.Scheduler.StopEverything) - - go testCtx.Scheduler.Run(testCtx.Ctx) - return testCtx }