From 782a092e2e291a3db2eafa5273e5753f24e7e604 Mon Sep 17 00:00:00 2001 From: Harry Zhang Date: Wed, 7 Feb 2018 10:20:45 -0800 Subject: [PATCH 1/3] Use feature gate in integration Clean up unused import --- test/integration/scheduler/extender_test.go | 18 +---- test/integration/scheduler/scheduler_test.go | 17 +---- test/integration/scheduler/taint_test.go | 18 +---- test/integration/scheduler/util.go | 74 +++++++++++++++----- test/integration/util/util.go | 19 +---- 5 files changed, 63 insertions(+), 83 deletions(-) diff --git a/test/integration/scheduler/extender_test.go b/test/integration/scheduler/extender_test.go index 6bb0154c82a..fc23f993364 100644 --- a/test/integration/scheduler/extender_test.go +++ b/test/integration/scheduler/extender_test.go @@ -41,7 +41,6 @@ import ( "k8s.io/kubernetes/pkg/scheduler" _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" - "k8s.io/kubernetes/pkg/scheduler/factory" e2e "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/integration/framework" ) @@ -364,22 +363,7 @@ func TestSchedulerExtender(t *testing.T) { policy.APIVersion = testapi.Groups[v1.GroupName].GroupVersion().String() informerFactory := informers.NewSharedInformerFactory(clientSet, 0) - schedulerConfigFactory := factory.NewConfigFactory( - v1.DefaultSchedulerName, - clientSet, - informerFactory.Core().V1().Nodes(), - informerFactory.Core().V1().Pods(), - informerFactory.Core().V1().PersistentVolumes(), - informerFactory.Core().V1().PersistentVolumeClaims(), - informerFactory.Core().V1().ReplicationControllers(), - informerFactory.Extensions().V1beta1().ReplicaSets(), - informerFactory.Apps().V1beta1().StatefulSets(), - informerFactory.Core().V1().Services(), - informerFactory.Policy().V1beta1().PodDisruptionBudgets(), - informerFactory.Storage().V1().StorageClasses(), - v1.DefaultHardPodAffinitySymmetricWeight, - enableEquivalenceCache, - ) + schedulerConfigFactory := CreateConfigurator(clientSet, informerFactory) schedulerConfig, err := schedulerConfigFactory.CreateFromConfig(policy) if err != nil { t.Fatalf("Couldn't create scheduler config: %v", err) diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index d29b9009f47..fbe5060de04 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -517,22 +517,7 @@ func TestMultiScheduler(t *testing.T) { informerFactory2 := informers.NewSharedInformerFactory(context.clientSet, 0) podInformer2 := factory.NewPodInformer(context.clientSet, 0, fooScheduler) - schedulerConfigFactory2 := factory.NewConfigFactory( - fooScheduler, - clientSet2, - informerFactory2.Core().V1().Nodes(), - podInformer2, - informerFactory2.Core().V1().PersistentVolumes(), - informerFactory2.Core().V1().PersistentVolumeClaims(), - informerFactory2.Core().V1().ReplicationControllers(), - informerFactory2.Extensions().V1beta1().ReplicaSets(), - informerFactory2.Apps().V1beta1().StatefulSets(), - informerFactory2.Core().V1().Services(), - informerFactory2.Policy().V1beta1().PodDisruptionBudgets(), - informerFactory2.Storage().V1().StorageClasses(), - v1.DefaultHardPodAffinitySymmetricWeight, - enableEquivalenceCache, - ) + schedulerConfigFactory2 := CreateConfiguratorWithPodInformer(fooScheduler, clientSet2, podInformer2, informerFactory2) schedulerConfig2, err := schedulerConfigFactory2.Create() if err != nil { t.Errorf("Couldn't create scheduler config: %v", err) diff --git a/test/integration/scheduler/taint_test.go b/test/integration/scheduler/taint_test.go index fb158f8e2ee..b0e027bab60 100644 --- a/test/integration/scheduler/taint_test.go +++ b/test/integration/scheduler/taint_test.go @@ -42,7 +42,6 @@ import ( "k8s.io/kubernetes/pkg/scheduler" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" - "k8s.io/kubernetes/pkg/scheduler/factory" "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction" pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" "k8s.io/kubernetes/test/integration/framework" @@ -113,22 +112,7 @@ func TestTaintNodeByCondition(t *testing.T) { algorithmprovider.ApplyFeatureGates() // Start scheduler - configurator := factory.NewConfigFactory( - v1.DefaultSchedulerName, - clientset, - informers.Core().V1().Nodes(), - informers.Core().V1().Pods(), - informers.Core().V1().PersistentVolumes(), - informers.Core().V1().PersistentVolumeClaims(), - informers.Core().V1().ReplicationControllers(), - informers.Extensions().V1beta1().ReplicaSets(), - informers.Apps().V1beta1().StatefulSets(), - informers.Core().V1().Services(), - informers.Policy().V1beta1().PodDisruptionBudgets(), - informers.Storage().V1().StorageClasses(), - v1.DefaultHardPodAffinitySymmetricWeight, - true, // Enable EqualCache by default. - ) + configurator := CreateConfigurator(clientset, informers) sched, err := scheduler.NewFromConfigurator(configurator, func(cfg *scheduler.Config) { cfg.StopEverything = controllerCh diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index 68e40eaf814..a1fe47455d8 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -27,7 +27,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" clientv1core "k8s.io/client-go/kubernetes/typed/core/v1" corelisters "k8s.io/client-go/listers/core/v1" @@ -35,6 +37,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/legacyscheme" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler" _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" "k8s.io/kubernetes/pkg/scheduler/factory" @@ -54,6 +57,60 @@ type TestContext struct { scheduler *scheduler.Scheduler } +// createConfigurator create a configurator for scheduler with given informer factory and default name. +func CreateConfigurator( + clientSet clientset.Interface, + informerFactory informers.SharedInformerFactory, +) scheduler.Configurator { + // Enable EnableEquivalenceClassCache for all integration tests. + utilfeature.DefaultFeatureGate.Set("EnableEquivalenceClassCache=true") + + return factory.NewConfigFactory( + v1.DefaultSchedulerName, + clientSet, + informerFactory.Core().V1().Nodes(), + informerFactory.Core().V1().Pods(), + informerFactory.Core().V1().PersistentVolumes(), + informerFactory.Core().V1().PersistentVolumeClaims(), + informerFactory.Core().V1().ReplicationControllers(), + informerFactory.Extensions().V1beta1().ReplicaSets(), + informerFactory.Apps().V1beta1().StatefulSets(), + informerFactory.Core().V1().Services(), + informerFactory.Policy().V1beta1().PodDisruptionBudgets(), + informerFactory.Storage().V1().StorageClasses(), + v1.DefaultHardPodAffinitySymmetricWeight, + utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache), + ) +} + +// CreateConfiguratorWithPodInformer create a configurator for scheduler with given informer factory, custom name and pod informer. +func CreateConfiguratorWithPodInformer( + schedulerName string, + clientSet clientset.Interface, + podInformer coreinformers.PodInformer, + informerFactory informers.SharedInformerFactory, +) scheduler.Configurator { + // Enable EnableEquivalenceClassCache for all integration tests. + utilfeature.DefaultFeatureGate.Set("EnableEquivalenceClassCache=true") + + return factory.NewConfigFactory( + schedulerName, + clientSet, + informerFactory.Core().V1().Nodes(), + podInformer, + informerFactory.Core().V1().PersistentVolumes(), + informerFactory.Core().V1().PersistentVolumeClaims(), + informerFactory.Core().V1().ReplicationControllers(), + informerFactory.Extensions().V1beta1().ReplicaSets(), + informerFactory.Apps().V1beta1().StatefulSets(), + informerFactory.Core().V1().Services(), + informerFactory.Policy().V1beta1().PodDisruptionBudgets(), + informerFactory.Storage().V1().StorageClasses(), + v1.DefaultHardPodAffinitySymmetricWeight, + utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache), + ) +} + // initTest initializes a test environment and creates a scheduler with default // configuration. func initTest(t *testing.T, nsPrefix string) *TestContext { @@ -66,22 +123,7 @@ func initTest(t *testing.T, nsPrefix string) *TestContext { context.clientSet = clientset.NewForConfigOrDie(&restclient.Config{Host: context.httpServer.URL}) context.informerFactory = informers.NewSharedInformerFactory(context.clientSet, 0) podInformer := factory.NewPodInformer(context.clientSet, 12*time.Hour, v1.DefaultSchedulerName) - context.schedulerConfigFactory = factory.NewConfigFactory( - v1.DefaultSchedulerName, - context.clientSet, - context.informerFactory.Core().V1().Nodes(), - podInformer, - context.informerFactory.Core().V1().PersistentVolumes(), - context.informerFactory.Core().V1().PersistentVolumeClaims(), - context.informerFactory.Core().V1().ReplicationControllers(), - context.informerFactory.Extensions().V1beta1().ReplicaSets(), - context.informerFactory.Apps().V1beta1().StatefulSets(), - context.informerFactory.Core().V1().Services(), - context.informerFactory.Policy().V1beta1().PodDisruptionBudgets(), - context.informerFactory.Storage().V1().StorageClasses(), - v1.DefaultHardPodAffinitySymmetricWeight, - true, - ) + context.schedulerConfigFactory = CreateConfiguratorWithPodInformer(v1.DefaultSchedulerName, context.clientSet, podInformer, context.informerFactory) var err error context.schedulerConfig, err = context.schedulerConfigFactory.Create() if err != nil { diff --git a/test/integration/util/util.go b/test/integration/util/util.go index ceb28e78b72..6357cb9753d 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -28,8 +28,8 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/scheduler" - "k8s.io/kubernetes/pkg/scheduler/factory" "k8s.io/kubernetes/test/integration/framework" + utilscheduler "k8s.io/kubernetes/test/integration/scheduler" ) // ShutdownFunc represents the function handle to be called, typically in a defer handler, to shutdown a running module @@ -63,22 +63,7 @@ func StartScheduler(clientSet clientset.Interface, enableEquivalenceCache bool) evtWatch := evtBroadcaster.StartRecordingToSink(&clientv1core.EventSinkImpl{ Interface: clientv1core.New(clientSet.CoreV1().RESTClient()).Events("")}) - schedulerConfigurator := factory.NewConfigFactory( - v1.DefaultSchedulerName, - clientSet, - informerFactory.Core().V1().Nodes(), - informerFactory.Core().V1().Pods(), - informerFactory.Core().V1().PersistentVolumes(), - informerFactory.Core().V1().PersistentVolumeClaims(), - informerFactory.Core().V1().ReplicationControllers(), - informerFactory.Extensions().V1beta1().ReplicaSets(), - informerFactory.Apps().V1beta1().StatefulSets(), - informerFactory.Core().V1().Services(), - informerFactory.Policy().V1beta1().PodDisruptionBudgets(), - informerFactory.Storage().V1().StorageClasses(), - v1.DefaultHardPodAffinitySymmetricWeight, - enableEquivalenceCache, - ) + schedulerConfigurator := utilscheduler.CreateConfigurator(clientSet, informerFactory) sched, err := scheduler.NewFromConfigurator(schedulerConfigurator, func(conf *scheduler.Config) { conf.Recorder = evtBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "scheduler"}) From 187c5bc91eea917e8d35b88fb4be606c678bd2fa Mon Sep 17 00:00:00 2001 From: Harry Zhang Date: Sun, 11 Mar 2018 18:07:10 -0700 Subject: [PATCH 2/3] Use initTest for integration to start scheduler Fix import path Fix configurator name --- test/integration/scheduler/extender_test.go | 34 +---- test/integration/scheduler/scheduler_test.go | 2 +- test/integration/scheduler/taint_test.go | 64 +++------ test/integration/scheduler/util.go | 133 ++++++++++++------ .../scheduler/volume_binding_test.go | 71 +--------- test/integration/util/util.go | 32 ++++- 6 files changed, 152 insertions(+), 184 deletions(-) diff --git a/test/integration/scheduler/extender_test.go b/test/integration/scheduler/extender_test.go index fc23f993364..4fc8cbfeb65 100644 --- a/test/integration/scheduler/extender_test.go +++ b/test/integration/scheduler/extender_test.go @@ -31,18 +31,11 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" - v1core "k8s.io/client-go/kubernetes/typed/core/v1" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/testapi" - "k8s.io/kubernetes/pkg/scheduler" _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" e2e "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/integration/framework" ) const ( @@ -287,13 +280,8 @@ func machine_3_Prioritizer(pod *v1.Pod, nodes *v1.NodeList) (*schedulerapi.HostP } func TestSchedulerExtender(t *testing.T) { - _, s, closeFn := framework.RunAMaster(nil) - defer closeFn() - - ns := framework.CreateTestingNamespace("scheduler-extender", s, t) - defer framework.DeleteTestingNamespace(ns, s, t) - - clientSet := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()}}) + context := initTestMaster(t, "scheduler-extender", nil) + clientSet := context.clientSet extender1 := &Extender{ name: "extender1", @@ -362,22 +350,10 @@ func TestSchedulerExtender(t *testing.T) { } policy.APIVersion = testapi.Groups[v1.GroupName].GroupVersion().String() - informerFactory := informers.NewSharedInformerFactory(clientSet, 0) - schedulerConfigFactory := CreateConfigurator(clientSet, informerFactory) - schedulerConfig, err := schedulerConfigFactory.CreateFromConfig(policy) - if err != nil { - t.Fatalf("Couldn't create scheduler config: %v", err) - } - eventBroadcaster := record.NewBroadcaster() - schedulerConfig.Recorder = eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: v1.DefaultSchedulerName}) - eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(clientSet.CoreV1().RESTClient()).Events("")}) - scheduler, _ := scheduler.NewFromConfigurator(&scheduler.FakeConfigurator{Config: schedulerConfig}, nil...) - informerFactory.Start(schedulerConfig.StopEverything) - scheduler.Run() + context = initTestScheduler(t, context, nil, false, &policy) + defer cleanupTest(t, context) - defer close(schedulerConfig.StopEverything) - - DoTestPodScheduling(ns, t, clientSet) + DoTestPodScheduling(context.ns, t, clientSet) } func DoTestPodScheduling(ns *v1.Namespace, t *testing.T, cs clientset.Interface) { diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index fbe5060de04..0f2b400d8a2 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -517,7 +517,7 @@ func TestMultiScheduler(t *testing.T) { informerFactory2 := informers.NewSharedInformerFactory(context.clientSet, 0) podInformer2 := factory.NewPodInformer(context.clientSet, 0, fooScheduler) - schedulerConfigFactory2 := CreateConfiguratorWithPodInformer(fooScheduler, clientSet2, podInformer2, informerFactory2) + schedulerConfigFactory2 := createConfiguratorWithPodInformer(fooScheduler, clientSet2, podInformer2, informerFactory2) schedulerConfig2, err := schedulerConfigFactory2.Create() if err != nil { t.Errorf("Couldn't create scheduler config: %v", err) diff --git a/test/integration/scheduler/taint_test.go b/test/integration/scheduler/taint_test.go index b0e027bab60..84181d9ad50 100644 --- a/test/integration/scheduler/taint_test.go +++ b/test/integration/scheduler/taint_test.go @@ -19,8 +19,6 @@ package scheduler // This file tests the Taint feature. import ( - "net/http" - "net/http/httptest" "reflect" "testing" "time" @@ -29,22 +27,17 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/informers" - clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" internalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" "k8s.io/kubernetes/pkg/controller/nodelifecycle" kubeadmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" - "k8s.io/kubernetes/pkg/scheduler" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction" pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" - "k8s.io/kubernetes/test/integration/framework" ) // TestTaintNodeByCondition verifies: @@ -52,37 +45,35 @@ import ( // 2. NodeController taints nodes by node condition // 3. Scheduler allows pod to tolerate node condition taints, e.g. network unavailable func TestTaintNodeByCondition(t *testing.T) { - h := &framework.MasterHolder{Initialized: make(chan struct{})} - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - <-h.Initialized - h.M.GenericAPIServer.Handler.ServeHTTP(w, req) - })) - // Enable TaintNodeByCondition utilfeature.DefaultFeatureGate.Set("TaintNodesByCondition=True") - // Build clientset and informers for controllers. - internalClientset := internalclientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()}}) - internalInformers := internalinformers.NewSharedInformerFactory(internalClientset, time.Second) - - clientset := clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()}}) - informers := informers.NewSharedInformerFactory(clientset, time.Second) - // Build PodToleration Admission. admission := podtolerationrestriction.NewPodTolerationsPlugin(&pluginapi.Configuration{}) + + context := initTestMaster(t, "default", admission) + + // Build clientset and informers for controllers. + internalClientset := internalclientset.NewForConfigOrDie(&restclient.Config{ + QPS: -1, + Host: context.httpServer.URL, + ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()}}) + internalInformers := internalinformers.NewSharedInformerFactory(internalClientset, time.Second) + kubeadmission.WantsInternalKubeClientSet(admission).SetInternalKubeClientSet(internalClientset) kubeadmission.WantsInternalKubeInformerFactory(admission).SetInternalKubeInformerFactory(internalInformers) - // Start master with admission. - masterConfig := framework.NewIntegrationTestMasterConfig() - masterConfig.GenericConfig.AdmissionControl = admission - _, _, closeFn := framework.RunAMasterUsingServer(masterConfig, s, h) - defer closeFn() - - nsName := "default" controllerCh := make(chan struct{}) defer close(controllerCh) + // Apply feature gates to enable TaintNodesByCondition + algorithmprovider.ApplyFeatureGates() + + context = initTestScheduler(t, context, controllerCh, false, nil) + clientset := context.clientSet + informers := context.informerFactory + nsName := context.ns.Name + // Start NodeLifecycleController for taint. nc, err := nodelifecycle.NewNodeLifecycleController( informers.Core().V1().Pods(), @@ -108,27 +99,8 @@ func TestTaintNodeByCondition(t *testing.T) { } go nc.Run(controllerCh) - // Apply feature gates to enable TaintNodesByCondition - algorithmprovider.ApplyFeatureGates() - - // Start scheduler - configurator := CreateConfigurator(clientset, informers) - - sched, err := scheduler.NewFromConfigurator(configurator, func(cfg *scheduler.Config) { - cfg.StopEverything = controllerCh - cfg.Recorder = &record.FakeRecorder{} - }) - if err != nil { - t.Errorf("Failed to create scheduler: %v.", err) - return - } - go sched.Run() - // Waiting for all controller sync. - informers.Start(controllerCh) internalInformers.Start(controllerCh) - - informers.WaitForCacheSync(controllerCh) internalInformers.WaitForCacheSync(controllerCh) // ------------------------------------------- diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index a1fe47455d8..c0acc007830 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -18,6 +18,7 @@ package scheduler import ( "fmt" + "net/http" "testing" "time" @@ -27,7 +28,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/admission" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" @@ -36,10 +39,12 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/api/testapi" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler" _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" + schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "k8s.io/kubernetes/pkg/scheduler/factory" "k8s.io/kubernetes/test/integration/framework" @@ -57,42 +62,13 @@ type TestContext struct { scheduler *scheduler.Scheduler } -// createConfigurator create a configurator for scheduler with given informer factory and default name. -func CreateConfigurator( - clientSet clientset.Interface, - informerFactory informers.SharedInformerFactory, -) scheduler.Configurator { - // Enable EnableEquivalenceClassCache for all integration tests. - utilfeature.DefaultFeatureGate.Set("EnableEquivalenceClassCache=true") - - return factory.NewConfigFactory( - v1.DefaultSchedulerName, - clientSet, - informerFactory.Core().V1().Nodes(), - informerFactory.Core().V1().Pods(), - informerFactory.Core().V1().PersistentVolumes(), - informerFactory.Core().V1().PersistentVolumeClaims(), - informerFactory.Core().V1().ReplicationControllers(), - informerFactory.Extensions().V1beta1().ReplicaSets(), - informerFactory.Apps().V1beta1().StatefulSets(), - informerFactory.Core().V1().Services(), - informerFactory.Policy().V1beta1().PodDisruptionBudgets(), - informerFactory.Storage().V1().StorageClasses(), - v1.DefaultHardPodAffinitySymmetricWeight, - utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache), - ) -} - -// CreateConfiguratorWithPodInformer create a configurator for scheduler with given informer factory, custom name and pod informer. -func CreateConfiguratorWithPodInformer( +// createConfiguratorWithPodInformer create a configurator for scheduler with given informer factory, custom name and pod informer. +func createConfiguratorWithPodInformer( schedulerName string, clientSet clientset.Interface, podInformer coreinformers.PodInformer, informerFactory informers.SharedInformerFactory, ) scheduler.Configurator { - // Enable EnableEquivalenceClassCache for all integration tests. - utilfeature.DefaultFeatureGate.Set("EnableEquivalenceClassCache=true") - return factory.NewConfigFactory( schedulerName, clientSet, @@ -111,35 +87,108 @@ func CreateConfiguratorWithPodInformer( ) } -// initTest initializes a test environment and creates a scheduler with default +// initTestMasterAndScheduler initializes a test environment and creates a master with default // configuration. -func initTest(t *testing.T, nsPrefix string) *TestContext { +func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface) *TestContext { var context TestContext + + // 1. Create master + h := &framework.MasterHolder{Initialized: make(chan struct{})} + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + <-h.Initialized + h.M.GenericAPIServer.Handler.ServeHTTP(w, req) + })) + masterConfig := framework.NewIntegrationTestMasterConfig() - _, context.httpServer, context.closeFn = framework.RunAMaster(masterConfig) - context.ns = framework.CreateTestingNamespace(nsPrefix+string(uuid.NewUUID()), context.httpServer, t) + if admission != nil { + masterConfig.GenericConfig.AdmissionControl = admission + } + + _, context.httpServer, context.closeFn = framework.RunAMasterUsingServer(masterConfig, s, h) + + if nsPrefix != "default" { + context.ns = framework.CreateTestingNamespace(nsPrefix+string(uuid.NewUUID()), s, t) + } else { + context.ns = framework.CreateTestingNamespace("default", s, t) + } + + // 2. Create kubeclient + context.clientSet = clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()}}) + return &context +} + +// initTestScheduler initializes a test environment and creates a scheduler with default +// configuration. +func initTestScheduler( + t *testing.T, + context *TestContext, + controllerCh chan struct{}, + setPodInformer bool, + policy *schedulerapi.Policy, +) *TestContext { + // Enable EnableEquivalenceClassCache for all integration tests. + defer utilfeaturetesting.SetFeatureGateDuringTest( + t, + utilfeature.DefaultFeatureGate, + features.EnableEquivalenceClassCache, true)() + + // 1. Create scheduler + context.informerFactory = informers.NewSharedInformerFactory(context.clientSet, time.Second) + + var podInformer coreinformers.PodInformer + + // create independent pod informer if required + if setPodInformer { + podInformer = factory.NewPodInformer(context.clientSet, 12*time.Hour, v1.DefaultSchedulerName) + } else { + podInformer = context.informerFactory.Core().V1().Pods() + } + + context.schedulerConfigFactory = createConfiguratorWithPodInformer( + v1.DefaultSchedulerName, context.clientSet, podInformer, context.informerFactory) - context.clientSet = clientset.NewForConfigOrDie(&restclient.Config{Host: context.httpServer.URL}) - context.informerFactory = informers.NewSharedInformerFactory(context.clientSet, 0) - podInformer := factory.NewPodInformer(context.clientSet, 12*time.Hour, v1.DefaultSchedulerName) - context.schedulerConfigFactory = CreateConfiguratorWithPodInformer(v1.DefaultSchedulerName, context.clientSet, podInformer, context.informerFactory) var err error - context.schedulerConfig, err = context.schedulerConfigFactory.Create() + + if policy != nil { + context.schedulerConfig, err = context.schedulerConfigFactory.CreateFromConfig(*policy) + } else { + context.schedulerConfig, err = context.schedulerConfigFactory.Create() + } + if err != nil { t.Fatalf("Couldn't create scheduler config: %v", err) } + + // set controllerCh if provided. + if controllerCh != nil { + context.schedulerConfig.StopEverything = controllerCh + } + + // set setPodInformer if provided. + if setPodInformer { + go podInformer.Informer().Run(context.schedulerConfig.StopEverything) + } + eventBroadcaster := record.NewBroadcaster() context.schedulerConfig.Recorder = eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: v1.DefaultSchedulerName}) eventBroadcaster.StartRecordingToSink(&clientv1core.EventSinkImpl{Interface: clientv1core.New(context.clientSet.CoreV1().RESTClient()).Events("")}) - go podInformer.Informer().Run(context.schedulerConfig.StopEverything) + context.informerFactory.Start(context.schedulerConfig.StopEverything) + context.informerFactory.WaitForCacheSync(context.schedulerConfig.StopEverything) + context.scheduler, err = scheduler.NewFromConfigurator(&scheduler.FakeConfigurator{Config: context.schedulerConfig}, nil...) if err != nil { t.Fatalf("Couldn't create scheduler: %v", err) } context.scheduler.Run() - return &context + return context +} + +// initTest initializes a test environment and creates master and scheduler with default +// configuration. +func initTest(t *testing.T, nsPrefix string) *TestContext { + return initTestScheduler(t, initTestMaster(t, nsPrefix, nil), nil, true, nil) } // cleanupTest deletes the scheduler and the test namespace. It should be called diff --git a/test/integration/scheduler/volume_binding_test.go b/test/integration/scheduler/volume_binding_test.go index 0c5c919b333..a3e0197fdcd 100644 --- a/test/integration/scheduler/volume_binding_test.go +++ b/test/integration/scheduler/volume_binding_test.go @@ -20,8 +20,6 @@ package scheduler import ( "fmt" - "net/http" - "net/http/httptest" "strconv" "strings" "testing" @@ -35,17 +33,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" - v1core "k8s.io/client-go/kubernetes/typed/core/v1" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" - "k8s.io/kubernetes/pkg/scheduler" - "k8s.io/kubernetes/pkg/scheduler/factory" - "k8s.io/kubernetes/test/integration/framework" ) type testConfig struct { @@ -347,27 +336,17 @@ func TestPVAffinityConflict(t *testing.T) { } func setupCluster(t *testing.T, nsName string, numberOfNodes int) *testConfig { - h := &framework.MasterHolder{Initialized: make(chan struct{})} - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - <-h.Initialized - h.M.GenericAPIServer.Handler.ServeHTTP(w, req) - })) - // Enable feature gates utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true,PersistentLocalVolumes=true") - // Build clientset and informers for controllers. - clientset := clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()}}) - informers := informers.NewSharedInformerFactory(clientset, time.Second) - - // Start master - masterConfig := framework.NewIntegrationTestMasterConfig() - - _, _, closeFn := framework.RunAMasterUsingServer(masterConfig, s, h) - ns := framework.CreateTestingNamespace(nsName, s, t).Name - controllerCh := make(chan struct{}) + context := initTestScheduler(t, initTestMaster(t, nsName, nil), controllerCh, false, nil) + + clientset := context.clientSet + ns := context.ns.Name + informers := context.informerFactory + // Start PV controller for volume binding. params := persistentvolume.ControllerParameters{ KubeClient: clientset, @@ -388,40 +367,6 @@ func setupCluster(t *testing.T, nsName string, numberOfNodes int) *testConfig { } go ctrl.Run(controllerCh) - // Start scheduler - configurator := factory.NewConfigFactory( - v1.DefaultSchedulerName, - clientset, - informers.Core().V1().Nodes(), - informers.Core().V1().Pods(), - informers.Core().V1().PersistentVolumes(), - informers.Core().V1().PersistentVolumeClaims(), - informers.Core().V1().ReplicationControllers(), - informers.Extensions().V1beta1().ReplicaSets(), - informers.Apps().V1beta1().StatefulSets(), - informers.Core().V1().Services(), - informers.Policy().V1beta1().PodDisruptionBudgets(), - informers.Storage().V1().StorageClasses(), - v1.DefaultHardPodAffinitySymmetricWeight, - true, // Enable EqualCache by default. - ) - - eventBroadcaster := record.NewBroadcaster() - sched, err := scheduler.NewFromConfigurator(configurator, func(cfg *scheduler.Config) { - cfg.StopEverything = controllerCh - cfg.Recorder = eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: v1.DefaultSchedulerName}) - eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(clientset.CoreV1().RESTClient()).Events("")}) - }) - if err != nil { - t.Fatalf("Failed to create scheduler: %v.", err) - } - - go sched.Run() - - // Waiting for all controller sync. - informers.Start(controllerCh) - informers.WaitForCacheSync(controllerCh) - // Create shared objects // Create nodes for i := 0; i < numberOfNodes; i++ { @@ -470,9 +415,7 @@ func setupCluster(t *testing.T, nsName string, numberOfNodes int) *testConfig { clientset.CoreV1().PersistentVolumeClaims(ns).DeleteCollection(nil, metav1.ListOptions{}) clientset.CoreV1().PersistentVolumes().DeleteCollection(nil, metav1.ListOptions{}) clientset.StorageV1().StorageClasses().DeleteCollection(nil, metav1.ListOptions{}) - clientset.CoreV1().Nodes().DeleteCollection(nil, metav1.ListOptions{}) - close(controllerCh) - closeFn() + cleanupTest(t, context) utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false,LocalPersistentVolumes=false") }, } diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 6357cb9753d..03426467133 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -22,14 +22,16 @@ import ( "github.com/golang/glog" "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" clientv1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler" + "k8s.io/kubernetes/pkg/scheduler/factory" "k8s.io/kubernetes/test/integration/framework" - utilscheduler "k8s.io/kubernetes/test/integration/scheduler" ) // ShutdownFunc represents the function handle to be called, typically in a defer handler, to shutdown a running module @@ -63,7 +65,7 @@ func StartScheduler(clientSet clientset.Interface, enableEquivalenceCache bool) evtWatch := evtBroadcaster.StartRecordingToSink(&clientv1core.EventSinkImpl{ Interface: clientv1core.New(clientSet.CoreV1().RESTClient()).Events("")}) - schedulerConfigurator := utilscheduler.CreateConfigurator(clientSet, informerFactory) + schedulerConfigurator := createSchedulerConfigurator(clientSet, informerFactory) sched, err := scheduler.NewFromConfigurator(schedulerConfigurator, func(conf *scheduler.Config) { conf.Recorder = evtBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "scheduler"}) @@ -86,3 +88,29 @@ func StartScheduler(clientSet clientset.Interface, enableEquivalenceCache bool) } return schedulerConfigurator, shutdownFunc } + +// createSchedulerConfigurator create a configurator for scheduler with given informer factory and default name. +func createSchedulerConfigurator( + clientSet clientset.Interface, + informerFactory informers.SharedInformerFactory, +) scheduler.Configurator { + // Enable EnableEquivalenceClassCache for all integration tests. + utilfeature.DefaultFeatureGate.Set("EnableEquivalenceClassCache=true") + + return factory.NewConfigFactory( + v1.DefaultSchedulerName, + clientSet, + informerFactory.Core().V1().Nodes(), + informerFactory.Core().V1().Pods(), + informerFactory.Core().V1().PersistentVolumes(), + informerFactory.Core().V1().PersistentVolumeClaims(), + informerFactory.Core().V1().ReplicationControllers(), + informerFactory.Extensions().V1beta1().ReplicaSets(), + informerFactory.Apps().V1beta1().StatefulSets(), + informerFactory.Core().V1().Services(), + informerFactory.Policy().V1beta1().PodDisruptionBudgets(), + informerFactory.Storage().V1().StorageClasses(), + v1.DefaultHardPodAffinitySymmetricWeight, + utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache), + ) +} From 2fae40d8868799846a50b7e33ccbd6270074d697 Mon Sep 17 00:00:00 2001 From: Harry Zhang Date: Sun, 11 Mar 2018 18:31:12 -0700 Subject: [PATCH 3/3] Update bazel --- test/integration/scheduler/BUILD | 7 +++++++ test/integration/util/BUILD | 2 ++ 2 files changed, 9 insertions(+) diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index 1906eaebd58..a1b294fa4fc 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -85,9 +85,12 @@ go_library( importpath = "k8s.io/kubernetes/test/integration/scheduler", deps = [ "//pkg/api/legacyscheme:go_default_library", + "//pkg/api/testapi:go_default_library", "//pkg/api/v1/pod:go_default_library", + "//pkg/features:go_default_library", "//pkg/scheduler:go_default_library", "//pkg/scheduler/algorithmprovider:go_default_library", + "//pkg/scheduler/api:go_default_library", "//pkg/scheduler/factory:go_default_library", "//test/integration/framework:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -96,7 +99,11 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", + "//vendor/k8s.io/client-go/informers/core/v1:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/listers/core/v1:go_default_library", diff --git a/test/integration/util/BUILD b/test/integration/util/BUILD index bfa500665c8..e6edbe9dcbe 100644 --- a/test/integration/util/BUILD +++ b/test/integration/util/BUILD @@ -16,12 +16,14 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", "//pkg/cloudprovider/providers/gce/cloud:go_default_library", + "//pkg/features:go_default_library", "//pkg/scheduler:go_default_library", "//pkg/scheduler/factory:go_default_library", "//test/integration/framework:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/oauth2:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",