From 3930362ad40301de070f0be5e3cd669a38a35062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Tyczy=C5=84ski?= Date: Fri, 10 Jun 2022 16:35:42 +0200 Subject: [PATCH] Clean(er) shutdown of apiserver integration tests --- test/integration/apiserver/apiserver_test.go | 179 +++++++++--------- .../apiserver/cve_2021_29923_test.go | 22 ++- test/integration/apiserver/export_test.go | 4 +- .../apiserver/no_new_betas_test.go | 4 +- test/integration/apiserver/patch_test.go | 13 +- test/integration/apiserver/print_test.go | 10 +- test/integration/apiserver/watchcache_test.go | 29 +-- 7 files changed, 137 insertions(+), 124 deletions(-) diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 5553e6a5b17..c5ed11ac5b2 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -24,7 +24,6 @@ import ( "io" "net" "net/http" - "net/http/httptest" "path" "reflect" "strconv" @@ -55,6 +54,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/handlers" "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/storage/storagebackend" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -64,52 +64,38 @@ import ( "k8s.io/client-go/tools/pager" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/klog/v2" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" - "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" ) -func setup(t testing.TB, groupVersions ...schema.GroupVersion) (*httptest.Server, clientset.Interface, framework.CloseFunc) { +func setup(t *testing.T, groupVersions ...schema.GroupVersion) (clientset.Interface, *restclient.Config, framework.TearDownFunc) { return setupWithResources(t, groupVersions, nil) } -func setupWithOptions(t testing.TB, opts *framework.ControlPlaneConfigOptions, groupVersions ...schema.GroupVersion) (*httptest.Server, clientset.Interface, framework.CloseFunc) { - return setupWithResourcesWithOptions(t, opts, groupVersions, nil) +func setupWithResources(t *testing.T, groupVersions []schema.GroupVersion, resources []schema.GroupVersionResource) (clientset.Interface, *restclient.Config, framework.TearDownFunc) { + return framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerConfig: func(config *controlplane.Config) { + if len(groupVersions) > 0 || len(resources) > 0 { + resourceConfig := controlplane.DefaultAPIResourceConfigSource() + resourceConfig.EnableVersions(groupVersions...) + resourceConfig.EnableResources(resources...) + config.ExtraConfig.APIResourceConfigSource = resourceConfig + } + }, + }) } -func setupWithResources(t testing.TB, groupVersions []schema.GroupVersion, resources []schema.GroupVersionResource) (*httptest.Server, clientset.Interface, framework.CloseFunc) { - return setupWithResourcesWithOptions(t, &framework.ControlPlaneConfigOptions{}, groupVersions, resources) -} - -func setupWithResourcesWithOptions(t testing.TB, opts *framework.ControlPlaneConfigOptions, groupVersions []schema.GroupVersion, resources []schema.GroupVersionResource) (*httptest.Server, clientset.Interface, framework.CloseFunc) { - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfigWithOptions(opts) - if len(groupVersions) > 0 || len(resources) > 0 { - resourceConfig := controlplane.DefaultAPIResourceConfigSource() - resourceConfig.EnableVersions(groupVersions...) - resourceConfig.EnableResources(resources...) - controlPlaneConfig.ExtraConfig.APIResourceConfigSource = resourceConfig - } - controlPlaneConfig.GenericConfig.OpenAPIConfig = framework.DefaultOpenAPIConfig() - _, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - - clientSet, err := clientset.NewForConfig(&restclient.Config{Host: s.URL, QPS: -1}) - if err != nil { - t.Fatalf("Error in create clientset: %v", err) - } - return s, clientSet, closeFn -} - -func verifyStatusCode(t *testing.T, verb, URL, body string, expectedStatusCode int) { +func verifyStatusCode(t *testing.T, transport http.RoundTripper, verb, URL, body string, expectedStatusCode int) { // We don't use the typed Go client to send this request to be able to verify the response status code. bodyBytes := bytes.NewReader([]byte(body)) req, err := http.NewRequest(verb, URL, bodyBytes) if err != nil { t.Fatalf("unexpected error: %v in sending req with verb: %s, URL: %s and body: %s", err, verb, URL, body) } - transport := http.DefaultTransport klog.Infof("Sending request: %v", req) resp, err := transport.RoundTrip(req) if err != nil { @@ -161,8 +147,8 @@ var cascDel = ` ` func Test4xxStatusCodeInvalidPatch(t *testing.T) { - _, client, closeFn := setup(t) - defer closeFn() + client, _, tearDownFn := setup(t) + defer tearDownFn() obj := []byte(`{ "apiVersion": "apps/v1", @@ -225,12 +211,10 @@ func Test4xxStatusCodeInvalidPatch(t *testing.T) { } func TestCacheControl(t *testing.T) { - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfigWithOptions(&framework.ControlPlaneConfigOptions{}) - controlPlaneConfig.GenericConfig.OpenAPIConfig = framework.DefaultOpenAPIConfig() - instanceConfig, _, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() + server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) + defer server.TearDownFn() - rt, err := restclient.TransportFor(instanceConfig.GenericAPIServer.LoopbackClientConfig) + rt, err := restclient.TransportFor(server.ClientConfig) if err != nil { t.Fatal(err) } @@ -254,7 +238,7 @@ func TestCacheControl(t *testing.T) { } for _, path := range paths { t.Run(path, func(t *testing.T) { - req, err := http.NewRequest("GET", instanceConfig.GenericAPIServer.LoopbackClientConfig.Host+path, nil) + req, err := http.NewRequest("GET", server.ClientConfig.Host+path, nil) if err != nil { t.Fatal(err) } @@ -272,13 +256,10 @@ func TestCacheControl(t *testing.T) { // Tests that the apiserver returns HSTS headers as expected. func TestHSTS(t *testing.T) { - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfigWithOptions(&framework.ControlPlaneConfigOptions{}) - controlPlaneConfig.GenericConfig.OpenAPIConfig = framework.DefaultOpenAPIConfig() - controlPlaneConfig.GenericConfig.HSTSDirectives = []string{"max-age=31536000", "includeSubDomains"} - instanceConfig, _, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--strict-transport-security-directives=max-age=31536000,includeSubDomains"}, framework.SharedEtcd()) + defer server.TearDownFn() - rt, err := restclient.TransportFor(instanceConfig.GenericAPIServer.LoopbackClientConfig) + rt, err := restclient.TransportFor(server.ClientConfig) if err != nil { t.Fatal(err) } @@ -302,7 +283,7 @@ func TestHSTS(t *testing.T) { } for _, path := range paths { t.Run(path, func(t *testing.T) { - req, err := http.NewRequest("GET", instanceConfig.GenericAPIServer.LoopbackClientConfig.Host+path, nil) + req, err := http.NewRequest("GET", server.ClientConfig.Host+path, nil) if err != nil { t.Fatal(err) } @@ -320,11 +301,16 @@ func TestHSTS(t *testing.T) { // Tests that the apiserver returns 202 status code as expected. func Test202StatusCode(t *testing.T) { - s, clientSet, closeFn := setup(t) - defer closeFn() + clientSet, kubeConfig, tearDownFn := setup(t) + defer tearDownFn() - ns := framework.CreateTestingNamespace("status-code", t) - defer framework.DeleteTestingNamespace(ns, t) + transport, err := restclient.TransportFor(kubeConfig) + if err != nil { + t.Fatal(err) + } + + ns := framework.CreateNamespaceOrDie(clientSet, "status-code", t) + defer framework.DeleteNamespaceOrDie(clientSet, ns, t) rsClient := clientSet.AppsV1().ReplicaSets(ns.Name) @@ -334,7 +320,7 @@ func Test202StatusCode(t *testing.T) { if err != nil { t.Fatalf("Failed to create rs: %v", err) } - verifyStatusCode(t, "DELETE", s.URL+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), "", 200) + verifyStatusCode(t, transport, "DELETE", kubeConfig.Host+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), "", 200) // 2. Create the resource with a finalizer so that the resource is not immediately deleted and then delete it without setting DeleteOptions. // Verify that the apiserver still returns 200 since DeleteOptions.OrphanDependents is not set. @@ -344,7 +330,7 @@ func Test202StatusCode(t *testing.T) { if err != nil { t.Fatalf("Failed to create rs: %v", err) } - verifyStatusCode(t, "DELETE", s.URL+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), "", 200) + verifyStatusCode(t, transport, "DELETE", kubeConfig.Host+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), "", 200) // 3. Create the resource and then delete it with DeleteOptions.OrphanDependents=false. // Verify that the server still returns 200 since the resource is immediately deleted. @@ -353,7 +339,7 @@ func Test202StatusCode(t *testing.T) { if err != nil { t.Fatalf("Failed to create rs: %v", err) } - verifyStatusCode(t, "DELETE", s.URL+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), cascDel, 200) + verifyStatusCode(t, transport, "DELETE", kubeConfig.Host+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), cascDel, 200) // 4. Create the resource with a finalizer so that the resource is not immediately deleted and then delete it with DeleteOptions.OrphanDependents=false. // Verify that the server returns 202 in this case. @@ -363,7 +349,7 @@ func Test202StatusCode(t *testing.T) { if err != nil { t.Fatalf("Failed to create rs: %v", err) } - verifyStatusCode(t, "DELETE", s.URL+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), cascDel, 202) + verifyStatusCode(t, transport, "DELETE", kubeConfig.Host+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), cascDel, 202) } var ( @@ -378,13 +364,18 @@ func TestListOptions(t *testing.T) { for _, watchCacheEnabled := range []bool{true, false} { t.Run(fmt.Sprintf("watchCacheEnabled=%t", watchCacheEnabled), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIListChunking, true)() - etcdOptions := framework.DefaultEtcdOptions() - etcdOptions.EnableWatchCache = watchCacheEnabled - _, clientSet, closeFn := setupWithOptions(t, &framework.ControlPlaneConfigOptions{EtcdOptions: etcdOptions}) - defer closeFn() - ns := framework.CreateTestingNamespace("list-options", t) - defer framework.DeleteTestingNamespace(ns, t) + var storageTransport *storagebackend.TransportConfig + clientSet, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + opts.Etcd.EnableWatchCache = watchCacheEnabled + storageTransport = &opts.Etcd.StorageConfig.Transport + }, + }) + defer tearDownFn() + + ns := framework.CreateNamespaceOrDie(clientSet, "list-options", t) + defer framework.DeleteNamespaceOrDie(clientSet, ns, t) rsClient := clientSet.AppsV1().ReplicaSets(ns.Name) @@ -410,7 +401,7 @@ func TestListOptions(t *testing.T) { } // compact some of the revision history in etcd so we can test "too old" resource versions - _, kvClient, err := integration.GetEtcdClients(etcdOptions.StorageConfig.Transport) + _, kvClient, err := integration.GetEtcdClients(*storageTransport) if err != nil { t.Fatal(err) } @@ -610,13 +601,16 @@ func TestListResourceVersion0(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIListChunking, true)() - etcdOptions := framework.DefaultEtcdOptions() - etcdOptions.EnableWatchCache = tc.watchCacheEnabled - _, clientSet, closeFn := setupWithOptions(t, &framework.ControlPlaneConfigOptions{EtcdOptions: etcdOptions}) - defer closeFn() - ns := framework.CreateTestingNamespace("list-paging", t) - defer framework.DeleteTestingNamespace(ns, t) + clientSet, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + opts.Etcd.EnableWatchCache = tc.watchCacheEnabled + }, + }) + defer tearDownFn() + + ns := framework.CreateNamespaceOrDie(clientSet, "list-paging", t) + defer framework.DeleteNamespaceOrDie(clientSet, ns, t) rsClient := clientSet.AppsV1().ReplicaSets(ns.Name) @@ -665,11 +659,11 @@ func TestListResourceVersion0(t *testing.T) { func TestAPIListChunking(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIListChunking, true)() - _, clientSet, closeFn := setup(t) - defer closeFn() + clientSet, _, tearDownFn := setup(t) + defer tearDownFn() - ns := framework.CreateTestingNamespace("list-paging", t) - defer framework.DeleteTestingNamespace(ns, t) + ns := framework.CreateNamespaceOrDie(clientSet, "list-paging", t) + defer framework.DeleteNamespaceOrDie(clientSet, ns, t) rsClient := clientSet.AppsV1().ReplicaSets(ns.Name) @@ -743,13 +737,13 @@ func makeSecret(name string) *v1.Secret { } func TestNameInFieldSelector(t *testing.T) { - _, clientSet, closeFn := setup(t) - defer closeFn() + clientSet, _, tearDownFn := setup(t) + defer tearDownFn() numNamespaces := 3 for i := 0; i < 3; i++ { - ns := framework.CreateTestingNamespace(fmt.Sprintf("ns%d", i), t) - defer framework.DeleteTestingNamespace(ns, t) + ns := framework.CreateNamespaceOrDie(clientSet, fmt.Sprintf("ns%d", i), t) + defer framework.DeleteNamespaceOrDie(clientSet, ns, t) _, err := clientSet.CoreV1().Secrets(ns.Name).Create(context.TODO(), makeSecret("foo"), metav1.CreateOptions{}) if err != nil { @@ -834,8 +828,8 @@ func TestMetadataClient(t *testing.T) { } defer tearDown() - s, clientset, closeFn := setup(t) - defer closeFn() + clientset, kubeConfig, tearDownFn := setup(t) + defer tearDownFn() apiExtensionClient, err := apiextensionsclient.NewForConfig(config) if err != nil { @@ -885,12 +879,15 @@ func TestMetadataClient(t *testing.T) { name: "list, get, patch, and delete via metadata client", want: func(t *testing.T) { ns := "metadata-builtin" + namespace := framework.CreateNamespaceOrDie(clientset, ns, t) + defer framework.DeleteNamespaceOrDie(clientset, namespace, t) + svc, err := clientset.CoreV1().Services(ns).Create(context.TODO(), &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-1", Annotations: map[string]string{"foo": "bar"}}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}, metav1.CreateOptions{}) if err != nil { t.Fatalf("unable to create service: %v", err) } - cfg := metadata.ConfigFor(&restclient.Config{Host: s.URL}) + cfg := metadata.ConfigFor(kubeConfig) wrapper := &callWrapper{} cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper { wrapper.nested = rt @@ -1025,6 +1022,9 @@ func TestMetadataClient(t *testing.T) { name: "watch via metadata client", want: func(t *testing.T) { ns := "metadata-watch" + namespace := framework.CreateNamespaceOrDie(clientset, ns, t) + defer framework.DeleteNamespaceOrDie(clientset, namespace, t) + svc, err := clientset.CoreV1().Services(ns).Create(context.TODO(), &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-2", Annotations: map[string]string{"foo": "bar"}}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}, metav1.CreateOptions{}) if err != nil { t.Fatalf("unable to create service: %v", err) @@ -1033,7 +1033,7 @@ func TestMetadataClient(t *testing.T) { t.Fatalf("unable to patch cr: %v", err) } - cfg := metadata.ConfigFor(&restclient.Config{Host: s.URL}) + cfg := metadata.ConfigFor(kubeConfig) wrapper := &callWrapper{} cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper { wrapper.nested = rt @@ -1162,8 +1162,8 @@ func TestAPICRDProtobuf(t *testing.T) { } defer tearDown() - s, _, closeFn := setup(t) - defer closeFn() + _, kubeConfig, tearDownFn := setup(t) + defer tearDownFn() apiExtensionClient, err := apiextensionsclient.NewForConfig(config) if err != nil { @@ -1337,7 +1337,7 @@ func TestAPICRDProtobuf(t *testing.T) { cfg := dynamic.ConfigFor(config) if len(group) == 0 { - cfg = dynamic.ConfigFor(&restclient.Config{Host: s.URL}) + cfg = dynamic.ConfigFor(kubeConfig) cfg.APIPath = "/api" } else { cfg.APIPath = "/apis" @@ -1376,9 +1376,11 @@ func TestGetSubresourcesAsTables(t *testing.T) { } defer tearDown() - s, clientset, closeFn := setup(t) - defer closeFn() - fmt.Printf("%#v\n", clientset) + clientset, kubeConfig, tearDownFn := setup(t) + defer tearDownFn() + + ns := framework.CreateNamespaceOrDie(clientset, testNamespace, t) + defer framework.DeleteNamespaceOrDie(clientset, ns, t) apiExtensionClient, err := apiextensionsclient.NewForConfig(config) if err != nil { @@ -1557,7 +1559,7 @@ func TestGetSubresourcesAsTables(t *testing.T) { cfg := dynamic.ConfigFor(config) if len(group) == 0 { - cfg = dynamic.ConfigFor(&restclient.Config{Host: s.URL}) + cfg = dynamic.ConfigFor(kubeConfig) cfg.APIPath = "/api" } else { cfg.APIPath = "/apis" @@ -1596,8 +1598,11 @@ func TestTransform(t *testing.T) { } defer tearDown() - s, clientset, closeFn := setup(t) - defer closeFn() + clientset, kubeConfig, tearDownFn := setup(t) + defer tearDownFn() + + ns := framework.CreateNamespaceOrDie(clientset, testNamespace, t) + defer framework.DeleteNamespaceOrDie(clientset, ns, t) apiExtensionClient, err := apiextensionsclient.NewForConfig(config) if err != nil { @@ -2171,7 +2176,7 @@ func TestTransform(t *testing.T) { cfg := dynamic.ConfigFor(config) if len(group) == 0 { - cfg = dynamic.ConfigFor(&restclient.Config{Host: s.URL}) + cfg = dynamic.ConfigFor(kubeConfig) cfg.APIPath = "/api" } else { cfg.APIPath = "/apis" diff --git a/test/integration/apiserver/cve_2021_29923_test.go b/test/integration/apiserver/cve_2021_29923_test.go index 89bdd0aad43..03ae3cd1ed6 100644 --- a/test/integration/apiserver/cve_2021_29923_test.go +++ b/test/integration/apiserver/cve_2021_29923_test.go @@ -25,7 +25,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" - restclient "k8s.io/client-go/rest" + clientset "k8s.io/client-go/kubernetes" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/framework" ) @@ -37,20 +38,23 @@ func gvr(g, v, r string) schema.GroupVersionResource { // Is it possible that exist more fields that can contain IPs, the test consider the most significative. // xref: https://issues.k8s.io/100895 func TestCanaryCVE_2021_29923(t *testing.T) { - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - _, server, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() + // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd()) + defer server.TearDownFn() - config := restclient.Config{Host: server.URL} + client, err := clientset.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("unexpected error creating client: %v", err) + } - dynamicClient, err := dynamic.NewForConfig(&config) + ns := framework.CreateNamespaceOrDie(client, "test-cve-2021-29923", t) + defer framework.DeleteNamespaceOrDie(client, ns, t) + + dynamicClient, err := dynamic.NewForConfig(server.ClientConfig) if err != nil { t.Fatalf("unexpected error creating dynamic client: %v", err) } - ns := framework.CreateTestingNamespace("test-cve-2021-29923", t) - defer framework.DeleteTestingNamespace(ns, t) - objects := map[schema.GroupVersionResource]string{ // k8s.io/kubernetes/pkg/api/v1 gvr("", "v1", "nodes"): `{"kind": "Node", "apiVersion": "v1", "metadata": {"name": "node1"}, "spec": {"unschedulable": true}, "status": {"addresses":[{"address":"172.18.0.012","type":"InternalIP"}]}}`, diff --git a/test/integration/apiserver/export_test.go b/test/integration/apiserver/export_test.go index 5c03355c840..a2ecbea8d2f 100644 --- a/test/integration/apiserver/export_test.go +++ b/test/integration/apiserver/export_test.go @@ -27,8 +27,8 @@ import ( // Tests that the apiserver rejects the export param func TestExportRejection(t *testing.T) { - _, clientSet, closeFn := setup(t) - defer closeFn() + clientSet, _, tearDownFn := setup(t) + defer tearDownFn() _, err := clientSet.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{Name: "export-fail"}, diff --git a/test/integration/apiserver/no_new_betas_test.go b/test/integration/apiserver/no_new_betas_test.go index af7a6e5196b..fc7fc9706c9 100644 --- a/test/integration/apiserver/no_new_betas_test.go +++ b/test/integration/apiserver/no_new_betas_test.go @@ -57,11 +57,11 @@ func TestNoNewBetaAPIsByDefault(t *testing.T) { // if you found this because you want to create an integration test for your new beta API, the method you're looking for // is this setupWithResources method and you need to pass the resource you want to enable into it. - _, kubeClient, closeFn := setupWithResources(t, + kubeClient, _, tearDownFn := setupWithResources(t, []schema.GroupVersion{}, []schema.GroupVersionResource{}, ) - defer closeFn() + defer tearDownFn() _, allResourceLists, err := kubeClient.Discovery().ServerGroupsAndResources() if err != nil { diff --git a/test/integration/apiserver/patch_test.go b/test/integration/apiserver/patch_test.go index e619f6ce5de..7a1c09d8285 100644 --- a/test/integration/apiserver/patch_test.go +++ b/test/integration/apiserver/patch_test.go @@ -35,11 +35,11 @@ import ( // Tests that the apiserver retries patches func TestPatchConflicts(t *testing.T) { - _, clientSet, closeFn := setup(t) - defer closeFn() + clientSet, _, tearDownFn := setup(t) + defer tearDownFn() - ns := framework.CreateTestingNamespace("status-code", t) - defer framework.DeleteTestingNamespace(ns, t) + ns := framework.CreateNamespaceOrDie(clientSet, "status-code", t) + defer framework.DeleteNamespaceOrDie(clientSet, ns, t) numOfConcurrentPatches := 100 @@ -64,7 +64,10 @@ func TestPatchConflicts(t *testing.T) { } // Create the object we're going to conflict on - clientSet.CoreV1().Secrets(ns.Name).Create(context.TODO(), secret, metav1.CreateOptions{}) + _, err := clientSet.CoreV1().Secrets(ns.Name).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } client := clientSet.CoreV1().RESTClient() successes := int32(0) diff --git a/test/integration/apiserver/print_test.go b/test/integration/apiserver/print_test.go index 0babb85d8ff..53aefe239a9 100644 --- a/test/integration/apiserver/print_test.go +++ b/test/integration/apiserver/print_test.go @@ -120,7 +120,7 @@ var missingHanlders = sets.NewString( ) func TestServerSidePrint(t *testing.T) { - s, _, closeFn := setupWithResources(t, + clientSet, kubeConfig, tearDownFn := setupWithResources(t, // additional groupversions needed for the test to run []schema.GroupVersion{ {Group: "discovery.k8s.io", Version: "v1"}, @@ -140,16 +140,16 @@ func TestServerSidePrint(t *testing.T) { }, []schema.GroupVersionResource{}, ) - defer closeFn() + defer tearDownFn() - ns := framework.CreateTestingNamespace("server-print", t) - defer framework.DeleteTestingNamespace(ns, t) + ns := framework.CreateNamespaceOrDie(clientSet, "server-print", t) + defer framework.DeleteNamespaceOrDie(clientSet, ns, t) tableParam := fmt.Sprintf("application/json;as=Table;g=%s;v=%s, application/json", metav1beta1.GroupName, metav1beta1.SchemeGroupVersion.Version) printer := newFakePrinter(printersinternal.AddHandlers) configFlags := genericclioptions.NewTestConfigFlags(). - WithClientConfig(clientcmd.NewDefaultClientConfig(*createKubeConfig(s.URL), &clientcmd.ConfigOverrides{})) + WithClientConfig(clientcmd.NewDefaultClientConfig(*createKubeConfig(kubeConfig.Host), &clientcmd.ConfigOverrides{})) restConfig, err := configFlags.ToRESTConfig() if err != nil { diff --git a/test/integration/apiserver/watchcache_test.go b/test/integration/apiserver/watchcache_test.go index 57786d62642..70a6b05d973 100644 --- a/test/integration/apiserver/watchcache_test.go +++ b/test/integration/apiserver/watchcache_test.go @@ -26,14 +26,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/pkg/controlplane/reconcilers" "k8s.io/kubernetes/test/integration/framework" ) // setup create kube-apiserver backed up by two separate etcds, // with one of them containing events and the other all other objects. -func multiEtcdSetup(t testing.TB) (clientset.Interface, framework.CloseFunc) { +func multiEtcdSetup(t *testing.T) (clientset.Interface, framework.CloseFunc) { etcdArgs := []string{"--experimental-watch-progress-notify-interval", "1s"} etcd0URL, stopEtcd0, err := framework.RunCustomEtcd("etcd_watchcache0", etcdArgs) if err != nil { @@ -51,25 +52,25 @@ func multiEtcdSetup(t testing.TB) (clientset.Interface, framework.CloseFunc) { etcdOptions.EtcdServersOverrides = []string{fmt.Sprintf("/events#%s", etcd1URL)} etcdOptions.EnableWatchCache = true - opts := framework.ControlPlaneConfigOptions{EtcdOptions: etcdOptions} - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfigWithOptions(&opts) - // Switch off endpoints reconciler to avoid unnecessary operations. - controlPlaneConfig.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType - _, s, stopAPIServer := framework.RunAnAPIServer(controlPlaneConfig) + clientSet, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + // Ensure we're using the same etcd across apiserver restarts. + opts.Etcd = etcdOptions + }, + ModifyServerConfig: func(config *controlplane.Config) { + // Switch off endpoints reconciler to avoid unnecessary operations. + config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType + }, + }) closeFn := func() { - stopAPIServer() + tearDownFn() stopEtcd1() stopEtcd0() } - clientSet, err := clientset.NewForConfig(&restclient.Config{Host: s.URL, QPS: -1}) - if err != nil { - t.Fatalf("Error in create clientset: %v", err) - } - // Wait for apiserver to be stabilized. - // Everything but default service creation is checked in RunAnAPIServer above by + // Everything but default service creation is checked in StartTestServer above by // waiting for post start hooks, so we just wait for default service to exist. // TODO(wojtek-t): Figure out less fragile way. ctx := context.Background()