From c802118e81ab7b100a1b5d9f6d93e3b7a86021a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Tyczy=C5=84ski?= Date: Fri, 20 May 2022 13:55:51 +0200 Subject: [PATCH] Update scheduler tests --- test/integration/framework/test_server.go | 3 +- test/integration/node/lifecycle_test.go | 12 ++--- .../scheduler/preemption/preemption_test.go | 8 ++- .../integration/scheduler/taint/taint_test.go | 10 ++-- test/integration/util/util.go | 49 +++++++++---------- 5 files changed, 34 insertions(+), 48 deletions(-) diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index b77ebf05729..558b527db3a 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -22,6 +22,7 @@ import ( "net/http" "os" "path" + "strings" "testing" "time" @@ -59,7 +60,7 @@ type TearDownFunc func() // StartTestServer runs a kube-apiserver, optionally calling out to the setup.ModifyServerRunOptions and setup.ModifyServerConfig functions func StartTestServer(t *testing.T, setup TestServerSetup) (client.Interface, *rest.Config, TearDownFunc) { - certDir, err := os.MkdirTemp("", "test-integration-"+t.Name()) + certDir, err := os.MkdirTemp("", "test-integration-"+strings.ReplaceAll(t.Name(), "/", "_")) if err != nil { t.Fatalf("Couldn't create temp dir: %v", err) } diff --git a/test/integration/node/lifecycle_test.go b/test/integration/node/lifecycle_test.go index 00778ff11d9..2c7190dbfc8 100644 --- a/test/integration/node/lifecycle_test.go +++ b/test/integration/node/lifecycle_test.go @@ -26,7 +26,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -123,10 +122,9 @@ func TestTaintBasedEvictions(t *testing.T) { testCtx := testutils.InitTestAPIServer(t, "taint-based-evictions", admission) // Build clientset and informers for controllers. - externalClientset := kubernetes.NewForConfigOrDie(&restclient.Config{ - QPS: -1, - Host: testCtx.HTTPServer.URL, - ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) + externalClientConfig := restclient.CopyConfig(testCtx.KubeConfig) + externalClientConfig.QPS = -1 + externalClientset := kubernetes.NewForConfigOrDie(externalClientConfig) externalInformers := informers.NewSharedInformerFactory(externalClientset, time.Second) podTolerations.SetExternalKubeClientSet(externalClientset) podTolerations.SetExternalKubeInformerFactory(externalInformers) @@ -134,10 +132,6 @@ func TestTaintBasedEvictions(t *testing.T) { testCtx = testutils.InitTestScheduler(t, testCtx) defer testutils.CleanupTest(t, testCtx) cs := testCtx.ClientSet - _, err := cs.CoreV1().Namespaces().Create(context.TODO(), testCtx.NS, metav1.CreateOptions{}) - if err != nil { - t.Errorf("Failed to create namespace %+v", err) - } // Start NodeLifecycleController for taint. nc, err := nodelifecycle.NewNodeLifecycleController( diff --git a/test/integration/scheduler/preemption/preemption_test.go b/test/integration/scheduler/preemption/preemption_test.go index 4c56b7ec44a..7f96768daa6 100644 --- a/test/integration/scheduler/preemption/preemption_test.go +++ b/test/integration/scheduler/preemption/preemption_test.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" @@ -621,10 +620,9 @@ func TestPodPriorityResolution(t *testing.T) { cs := testCtx.ClientSet // Build clientset and informers for controllers. - externalClientset := kubernetes.NewForConfigOrDie(&restclient.Config{ - QPS: -1, - Host: testCtx.HTTPServer.URL, - ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) + externalClientConfig := restclient.CopyConfig(testCtx.KubeConfig) + externalClientConfig.QPS = -1 + externalClientset := kubernetes.NewForConfigOrDie(externalClientConfig) externalInformers := informers.NewSharedInformerFactory(externalClientset, time.Second) admission.SetExternalKubeClientSet(externalClientset) admission.SetExternalKubeInformerFactory(externalInformers) diff --git a/test/integration/scheduler/taint/taint_test.go b/test/integration/scheduler/taint/taint_test.go index dabdfbd6028..4805797a8ff 100644 --- a/test/integration/scheduler/taint/taint_test.go +++ b/test/integration/scheduler/taint/taint_test.go @@ -27,7 +27,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" @@ -68,13 +67,12 @@ func TestTaintNodeByCondition(t *testing.T) { // Build PodToleration Admission. admission := podtolerationrestriction.NewPodTolerationsPlugin(&pluginapi.Configuration{}) - testCtx := testutils.InitTestAPIServer(t, "default", admission) + testCtx := testutils.InitTestAPIServer(t, "taint-node-by-condition", admission) // Build clientset and informers for controllers. - externalClientset := kubernetes.NewForConfigOrDie(&restclient.Config{ - QPS: -1, - Host: testCtx.HTTPServer.URL, - ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) + externalClientConfig := restclient.CopyConfig(testCtx.KubeConfig) + externalClientConfig.QPS = -1 + externalClientset := kubernetes.NewForConfigOrDie(externalClientConfig) externalInformers := informers.NewSharedInformerFactory(externalClientset, 0) admission.SetExternalKubeClientSet(externalClientset) diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 16ca1f30af7..eee74da29fe 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "net/http/httptest" "testing" "time" @@ -28,7 +27,6 @@ import ( policy "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/admission" @@ -47,8 +45,10 @@ import ( pvutil "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" "k8s.io/kube-scheduler/config/v1beta3" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/controller/disruption" + "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/pkg/scheduler" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing" @@ -162,8 +162,7 @@ func StartFakePVController(clientSet clientset.Interface) ShutdownFunc { // TestContext store necessary context info type TestContext struct { - CloseFn framework.CloseFunc - HTTPServer *httptest.Server + CloseFn framework.TearDownFunc NS *v1.Namespace ClientSet clientset.Interface KubeConfig *restclient.Config @@ -341,32 +340,23 @@ func InitTestAPIServer(t *testing.T, nsPrefix string, admission admission.Interf CancelFn: cancelFunc, } - // 1. Create API server - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - - if admission != nil { - controlPlaneConfig.GenericConfig.AdmissionControl = admission - } - - _, testCtx.HTTPServer, testCtx.CloseFn = framework.RunAnAPIServer(controlPlaneConfig) + testCtx.ClientSet, testCtx.KubeConfig, testCtx.CloseFn = framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerRunOptions: func(options *options.ServerRunOptions) { + options.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount", "TaintNodesByCondition", "Priority"} + }, + ModifyServerConfig: func(config *controlplane.Config) { + if admission != nil { + config.GenericConfig.AdmissionControl = admission + } + }, + }) if nsPrefix != "default" { - testCtx.NS = framework.CreateTestingNamespace(nsPrefix+string(uuid.NewUUID()), t) + testCtx.NS = framework.CreateNamespaceOrDie(testCtx.ClientSet, nsPrefix+string(uuid.NewUUID()), t) } else { - testCtx.NS = framework.CreateTestingNamespace("default", t) + testCtx.NS = framework.CreateNamespaceOrDie(testCtx.ClientSet, "default", t) } - // 2. Create kubeclient - kubeConfig := &restclient.Config{ - QPS: 100, - Burst: 100, - Host: testCtx.HTTPServer.URL, - ContentConfig: restclient.ContentConfig{ - GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}, - }, - } - testCtx.KubeConfig = kubeConfig - testCtx.ClientSet = clientset.NewForConfigOrDie(kubeConfig) return &testCtx } @@ -428,6 +418,11 @@ func InitTestSchedulerWithOptions( eventBroadcaster.StartRecordingToSink(testCtx.Ctx.Done()) + oldCloseFn := testCtx.CloseFn + testCtx.CloseFn = func() { + oldCloseFn() + eventBroadcaster.Shutdown() + } return testCtx } @@ -466,9 +461,9 @@ func InitDisruptionController(t *testing.T, testCtx *TestContext) *disruption.Di discoveryClient := cacheddiscovery.NewMemCacheClient(testCtx.ClientSet.Discovery()) mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) - config := restclient.Config{Host: testCtx.HTTPServer.URL} + config := restclient.CopyConfig(testCtx.KubeConfig) scaleKindResolver := scale.NewDiscoveryScaleKindResolver(testCtx.ClientSet.Discovery()) - scaleClient, err := scale.NewForConfig(&config, mapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) + scaleClient, err := scale.NewForConfig(config, mapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) if err != nil { t.Fatalf("Error in create scaleClient: %v", err) }