From d150f15d912c37fd93aa0d2c3d2bfbf136f6940f Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 30 Aug 2019 22:42:38 -0400 Subject: [PATCH 1/4] Add readiness probes to CRD/Admission webhook pods --- test/e2e/apimachinery/crd_conversion_webhook.go | 12 ++++++++++++ test/e2e/apimachinery/webhook.go | 12 ++++++++++++ test/utils/image/manifest.go | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go index f6a28daa78c..ecc0df82ecf 100644 --- a/test/e2e/apimachinery/crd_conversion_webhook.go +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -292,6 +292,18 @@ func deployCustomResourceWebhookAndService(f *framework.Framework, image string, // Use a non-default port for containers. fmt.Sprintf("--port=%d", containerPort), }, + ReadinessProbe: &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Scheme: v1.URISchemeHTTPS, + Port: intstr.FromInt(int(containerPort)), + Path: "/readyz", + }, + }, + PeriodSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 30, + }, Image: image, Ports: []v1.ContainerPort{{ContainerPort: containerPort}}, }, diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index 4e28765ee23..177d398b15a 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -778,6 +778,18 @@ func deployWebhookAndService(f *framework.Framework, image string, context *cert // Use a non-default port for containers. fmt.Sprintf("--port=%d", containerPort), }, + ReadinessProbe: &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Scheme: v1.URISchemeHTTPS, + Port: intstr.FromInt(int(containerPort)), + Path: "/readyz", + }, + }, + PeriodSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 30, + }, Image: image, Ports: []v1.ContainerPort{{ContainerPort: containerPort}}, }, diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index b149f54a939..ee2d41922e9 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -204,7 +204,7 @@ const ( func initImageConfigs() map[int]Config { configs := map[int]Config{} - configs[Agnhost] = Config{e2eRegistry, "agnhost", "2.5"} + configs[Agnhost] = Config{e2eRegistry, "agnhost", "2.6"} configs[Alpine] = Config{dockerLibraryRegistry, "alpine", "3.7"} configs[AuthenticatedAlpine] = Config{gcAuthenticatedRegistry, "alpine", "3.7"} configs[AuthenticatedWindowsNanoServer] = Config{gcAuthenticatedRegistry, "windows-nanoserver", "v1"} From 97ad87df02f198053e8604452046c7c7e409dfcc Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 30 Aug 2019 22:43:21 -0400 Subject: [PATCH 2/4] Make CRD admission webhook e2e work in parallel test environments --- test/e2e/apimachinery/webhook.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index 177d398b15a..7e92f72927f 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -2061,8 +2061,8 @@ func registerValidatingWebhookForCRD(f *framework.Framework, configName string, }, SideEffects: &sideEffectsNone, AdmissionReviewVersions: []string{"v1", "v1beta1"}, - // Scope the webhook to just this namespace - NamespaceSelector: &metav1.LabelSelector{ + // Scope the webhook to just this test + ObjectSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, @@ -2113,6 +2113,9 @@ func testCRDDenyWebhook(f *framework.Framework) { ObjectMeta: metav1.ObjectMeta{ Name: name + "s." + group, Labels: map[string]string{ + // this label ensures our object is routed to this test's webhook + f.UniqueName: "true", + // this is the label the webhook disallows "webhook-e2e-test": "webhook-disallow", }, }, From 21a2951375ef6c7b95f440bff6e3eb26ffb473c7 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 30 Aug 2019 22:43:42 -0400 Subject: [PATCH 3/4] Make admission webhook e2e tests robust in non-aggregator-routing clusters --- test/e2e/apimachinery/webhook.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index 7e92f72927f..bd7781c0a26 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -419,6 +419,7 @@ var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() { }, Webhooks: []admissionregistrationv1.ValidatingWebhook{ newDenyConfigMapWebhookFixture(f, context, servicePort), + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "Creating validating webhook configuration") @@ -426,6 +427,8 @@ var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() { err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(hook.Name, nil) framework.ExpectNoError(err, "Deleting validating webhook configuration") }() + // ensure backend is ready before proceeding + waitWebhookConfigurationReady(f) ginkgo.By("Creating a configMap that does not comply to the validation webhook rules") err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f) @@ -509,6 +512,7 @@ var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() { }, Webhooks: []admissionregistrationv1.MutatingWebhook{ newMutateConfigMapWebhookFixture(f, context, 1, servicePort), + newMutatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "Creating mutating webhook configuration") @@ -516,7 +520,8 @@ var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() { err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(hook.Name, nil) framework.ExpectNoError(err, "Deleting mutating webhook configuration") }() - + // ensure backend is ready before proceeding + waitWebhookConfigurationReady(f) hook, err = admissionClient.MutatingWebhookConfigurations().Get(f.UniqueName, metav1.GetOptions{}) framework.ExpectNoError(err, "Getting mutating webhook configuration") ginkgo.By("Updating a mutating webhook configuration's rules to not include the create operation") @@ -2449,6 +2454,7 @@ func waitWebhookConfigurationReady(f *framework.Framework) error { // with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready. func newValidatingIsReadyWebhookFixture(f *framework.Framework, context *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook { sideEffectsNone := admissionregistrationv1.SideEffectClassNone + failOpen := admissionregistrationv1.Ignore return admissionregistrationv1.ValidatingWebhook{ Name: "validating-is-webhook-configuration-ready.k8s.io", Rules: []admissionregistrationv1.RuleWithOperations{{ @@ -2468,6 +2474,8 @@ func newValidatingIsReadyWebhookFixture(f *framework.Framework, context *certCon }, CABundle: context.signingCert, }, + // network failures while the service network routing is being set up should be ignored by the marker + FailurePolicy: &failOpen, SideEffects: &sideEffectsNone, AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just the markers namespace @@ -2485,6 +2493,7 @@ func newValidatingIsReadyWebhookFixture(f *framework.Framework, context *certCon // with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready. func newMutatingIsReadyWebhookFixture(f *framework.Framework, context *certContext, servicePort int32) admissionregistrationv1.MutatingWebhook { sideEffectsNone := admissionregistrationv1.SideEffectClassNone + failOpen := admissionregistrationv1.Ignore return admissionregistrationv1.MutatingWebhook{ Name: "mutating-is-webhook-configuration-ready.k8s.io", Rules: []admissionregistrationv1.RuleWithOperations{{ @@ -2504,6 +2513,8 @@ func newMutatingIsReadyWebhookFixture(f *framework.Framework, context *certConte }, CABundle: context.signingCert, }, + // network failures while the service network routing is being set up should be ignored by the marker + FailurePolicy: &failOpen, SideEffects: &sideEffectsNone, AdmissionReviewVersions: []string{"v1", "v1beta1"}, // Scope the webhook to just the markers namespace From 2d9f46f0732fd440081f721e6fcd519e69cc7595 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 30 Aug 2019 22:41:47 -0400 Subject: [PATCH 4/4] Make CRD conversion e2e tests robust in non-aggregator-routing clusters --- .../apimachinery/crd_conversion_webhook.go | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go index ecc0df82ecf..730bfa3ccef 100644 --- a/test/e2e/apimachinery/crd_conversion_webhook.go +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -167,6 +168,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Privileged:ClusterAdmin]", return } defer testcrd.CleanUp() + waitWebhookConversionReady(f, testcrd.Crd, testcrd.DynamicClients, "v2") testCustomResourceConversionWebhook(f, testcrd.Crd, testcrd.DynamicClients) }) @@ -201,6 +203,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Privileged:ClusterAdmin]", return } defer testcrd.CleanUp() + waitWebhookConversionReady(f, testcrd.Crd, testcrd.DynamicClients, "v2") testCRListConversion(f, testcrd) }) }) @@ -485,3 +488,29 @@ func testCRListConversion(f *framework.Framework, testCrd *crd.TestCrd) { verifyV2Object(f, crd, &list.Items[0]) verifyV2Object(f, crd, &list.Items[1]) } + +// waitWebhookConversionReady sends stub custom resource creation requests requiring conversion until one succeeds. +func waitWebhookConversionReady(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface, version string) { + framework.ExpectNoError(wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { + crInstance := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": crd.Spec.Names.Kind, + "apiVersion": crd.Spec.Group + "/" + version, + "metadata": map[string]interface{}{ + "name": f.UniqueName, + "namespace": f.Namespace.Name, + }, + }, + } + _, err := customResourceClients[version].Create(crInstance, metav1.CreateOptions{}) + if err != nil { + // tolerate clusters that do not set --enable-aggregator-routing and have to wait for kube-proxy + // to program the service network, during which conversion requests return errors + e2elog.Logf("error waiting for conversion to succeed during setup: %v", err) + return false, nil + } + + framework.ExpectNoError(customResourceClients[version].Delete(crInstance.GetName(), nil), "cleaning up stub object") + return true, nil + })) +}