From 28f88c91ee15fca2e2329b0a871f875d87bb11b3 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 13 May 2019 11:07:02 +0200 Subject: [PATCH 1/2] integration: Start{RealMasterOrDie->TestServer} in admissionwebhook tests --- .../apiserver/admissionwebhook/BUILD | 3 +- .../admissionwebhook/admission_test.go | 29 ++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/test/integration/apiserver/admissionwebhook/BUILD b/test/integration/apiserver/admissionwebhook/BUILD index 30236402e85..c59a47eabd8 100644 --- a/test/integration/apiserver/admissionwebhook/BUILD +++ b/test/integration/apiserver/admissionwebhook/BUILD @@ -12,7 +12,6 @@ go_test( "integration", ], deps = [ - "//cmd/kube-apiserver/app/options:go_default_library", "//cmd/kube-apiserver/app/testing:go_default_library", "//staging/src/k8s.io/api/admission/v1beta1:go_default_library", "//staging/src/k8s.io/api/admissionregistration/v1beta1:go_default_library", @@ -21,6 +20,7 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", @@ -31,6 +31,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", diff --git a/test/integration/apiserver/admissionwebhook/admission_test.go b/test/integration/apiserver/admissionwebhook/admission_test.go index bd506e4dcf1..aab139efa90 100644 --- a/test/integration/apiserver/admissionwebhook/admission_test.go +++ b/test/integration/apiserver/admissionwebhook/admission_test.go @@ -36,6 +36,7 @@ import ( corev1 "k8s.io/api/core/v1" extensionsv1beta1 "k8s.io/api/extensions/v1beta1" policyv1beta1 "k8s.io/api/policy/v1beta1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -46,9 +47,11 @@ import ( "k8s.io/apimachinery/pkg/util/wait" dynamic "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/etcd" + "k8s.io/kubernetes/test/integration/framework" ) const ( @@ -323,25 +326,23 @@ func TestWebhookV1beta1(t *testing.T) { defer webhookServer.Close() // start API server - master := etcd.StartRealMasterOrDie(t, func(opts *options.ServerRunOptions) { - // turn off admission plugins that add finalizers - opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount", "StorageObjectInUseProtection"} + s, err := kubeapiservertesting.StartTestServer(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{ + "--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection", + "--runtime-config=extensions/v1beta1/deployments=true,extensions/v1beta1/daemonsets=true,extensions/v1beta1/replicasets=true,extensions/v1beta1/podsecuritypolicies=true,extensions/v1beta1/networkpolicies=true", + }, framework.SharedEtcd()) + if err != nil { + t.Fatal(err) + } + defer s.TearDownFn() - // force enable all resources so we can check storage. - // TODO: drop these once we stop allowing them to be served. - opts.APIEnablement.RuntimeConfig["extensions/v1beta1/deployments"] = "true" - opts.APIEnablement.RuntimeConfig["extensions/v1beta1/daemonsets"] = "true" - opts.APIEnablement.RuntimeConfig["extensions/v1beta1/replicasets"] = "true" - opts.APIEnablement.RuntimeConfig["extensions/v1beta1/podsecuritypolicies"] = "true" - opts.APIEnablement.RuntimeConfig["extensions/v1beta1/networkpolicies"] = "true" - }) - defer master.Cleanup() + // create CRDs so we can make sure that custom resources do not get lost + etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(s.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...) // Configure a client with a distinct user name so that it is easy to distinguish requests // made by the client from requests made by controllers. We use this to filter out requests // before recording them to ensure we don't accidentally mistake requests from controllers // as requests made by the client. - clientConfig := master.Config + clientConfig := rest.CopyConfig(s.ClientConfig) clientConfig.Impersonate.UserName = testClientUsername clientConfig.Impersonate.Groups = []string{"system:masters", "system:authenticated"} client, err := clientset.NewForConfig(clientConfig) From 76b5bee48ca4605fb9ff2be74cdfaa630c743308 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 13 May 2019 11:29:40 +0200 Subject: [PATCH 2/2] integration: Start{RealMasterOrDie->TestServer} in dryrun tests --- test/integration/dryrun/BUILD | 3 ++ test/integration/dryrun/dryrun_test.go | 41 ++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/test/integration/dryrun/BUILD b/test/integration/dryrun/BUILD index e847eda66c2..5bfe9dc37ed 100644 --- a/test/integration/dryrun/BUILD +++ b/test/integration/dryrun/BUILD @@ -17,7 +17,9 @@ go_test( "integration", ], deps = [ + "//cmd/kube-apiserver/app/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", @@ -27,6 +29,7 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", diff --git a/test/integration/dryrun/dryrun_test.go b/test/integration/dryrun/dryrun_test.go index d487ef91b84..38e2e254929 100644 --- a/test/integration/dryrun/dryrun_test.go +++ b/test/integration/dryrun/dryrun_test.go @@ -19,7 +19,8 @@ package dryrun import ( "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -29,8 +30,11 @@ import ( "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" featuregatetesting "k8s.io/component-base/featuregate/testing" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/etcd" + "k8s.io/kubernetes/test/integration/framework" ) // Only add kinds to this list when this a virtual resource with get and create verbs that doesn't actually @@ -206,10 +210,29 @@ func DryRunDeleteTest(t *testing.T, rsc dynamic.ResourceInterface, name string) func TestDryRun(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DryRun, true)() - master := etcd.StartRealMasterOrDie(t) - defer master.Cleanup() + // start API server + s, err := kubeapiservertesting.StartTestServer(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{ + "--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection", + "--runtime-config=extensions/v1beta1/deployments=true,extensions/v1beta1/daemonsets=true,extensions/v1beta1/replicasets=true,extensions/v1beta1/podsecuritypolicies=true,extensions/v1beta1/networkpolicies=true", + }, framework.SharedEtcd()) + if err != nil { + t.Fatal(err) + } + defer s.TearDownFn() - if _, err := master.Client.CoreV1().Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}); err != nil { + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatal(err) + } + dynamicClient, err := dynamic.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatal(err) + } + + // create CRDs so we can make sure that custom resources do not get lost + etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(s.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...) + + if _, err := client.CoreV1().Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}); err != nil { t.Fatal(err) } @@ -225,7 +248,13 @@ func TestDryRun(t *testing.T) { dryrunData[resource] = data } - for _, resourceToTest := range master.Resources { + // gather resources to test + _, resources, err := client.Discovery().ServerGroupsAndResources() + if err != nil { + t.Fatalf("Failed to get ServerGroupsAndResources with error: %+v", err) + } + + for _, resourceToTest := range etcd.GetResources(t, resources) { t.Run(resourceToTest.Mapping.Resource.String(), func(t *testing.T) { mapping := resourceToTest.Mapping gvk := resourceToTest.Mapping.GroupVersionKind @@ -242,7 +271,7 @@ func TestDryRun(t *testing.T) { t.Fatalf("no test data for %s. Please add a test for your new type to etcd.GetEtcdStorageData().", gvResource) } - rsc, obj, err := etcd.JSONToUnstructured(testData.Stub, testNamespace, mapping, master.Dynamic) + rsc, obj, err := etcd.JSONToUnstructured(testData.Stub, testNamespace, mapping, dynamicClient) if err != nil { t.Fatalf("failed to unmarshal stub (%v): %v", testData.Stub, err) }