Merge pull request #82200 from liggitt/e2e-readyz-check

Make webhook/crd e2e tests behave in parallel and non-`--enable-aggregator-routing` test environments

test/e2e/apimachinery/crd_conversion_webhook.go

@@ -30,6 +30,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -167,6 +168,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Privileged:ClusterAdmin]",
 			return
 		}
 		defer testcrd.CleanUp()
+		waitWebhookConversionReady(f, testcrd.Crd, testcrd.DynamicClients, "v2")
 		testCustomResourceConversionWebhook(f, testcrd.Crd, testcrd.DynamicClients)
 	})
 
@@ -201,6 +203,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Privileged:ClusterAdmin]",
 			return
 		}
 		defer testcrd.CleanUp()
+		waitWebhookConversionReady(f, testcrd.Crd, testcrd.DynamicClients, "v2")
 		testCRListConversion(f, testcrd)
 	})
 })
@@ -292,6 +295,18 @@ func deployCustomResourceWebhookAndService(f *framework.Framework, image string,
 				// Use a non-default port for containers.
 				fmt.Sprintf("--port=%d", containerPort),
 			},
+			ReadinessProbe: &v1.Probe{
+				Handler: v1.Handler{
+					HTTPGet: &v1.HTTPGetAction{
+						Scheme: v1.URISchemeHTTPS,
+						Port:   intstr.FromInt(int(containerPort)),
+						Path:   "/readyz",
+					},
+				},
+				PeriodSeconds:    1,
+				SuccessThreshold: 1,
+				FailureThreshold: 30,
+			},
 			Image: image,
 			Ports: []v1.ContainerPort{{ContainerPort: containerPort}},
 		},
@@ -473,3 +488,29 @@ func testCRListConversion(f *framework.Framework, testCrd *crd.TestCrd) {
 	verifyV2Object(f, crd, &list.Items[0])
 	verifyV2Object(f, crd, &list.Items[1])
 }
+
+// waitWebhookConversionReady sends stub custom resource creation requests requiring conversion until one succeeds.
+func waitWebhookConversionReady(f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface, version string) {
+	framework.ExpectNoError(wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
+		crInstance := &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"kind":       crd.Spec.Names.Kind,
+				"apiVersion": crd.Spec.Group + "/" + version,
+				"metadata": map[string]interface{}{
+					"name":      f.UniqueName,
+					"namespace": f.Namespace.Name,
+				},
+			},
+		}
+		_, err := customResourceClients[version].Create(crInstance, metav1.CreateOptions{})
+		if err != nil {
+			// tolerate clusters that do not set --enable-aggregator-routing and have to wait for kube-proxy
+			// to program the service network, during which conversion requests return errors
+			e2elog.Logf("error waiting for conversion to succeed during setup: %v", err)
+			return false, nil
+		}
+
+		framework.ExpectNoError(customResourceClients[version].Delete(crInstance.GetName(), nil), "cleaning up stub object")
+		return true, nil
+	}))
+}

test/e2e/apimachinery/webhook.go

@@ -419,6 +419,7 @@ var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() {
 			},
 			Webhooks: []admissionregistrationv1.ValidatingWebhook{
 				newDenyConfigMapWebhookFixture(f, context, servicePort),
+				newValidatingIsReadyWebhookFixture(f, context, servicePort),
 			},
 		})
 		framework.ExpectNoError(err, "Creating validating webhook configuration")
@@ -426,6 +427,8 @@ var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() {
 			err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(hook.Name, nil)
 			framework.ExpectNoError(err, "Deleting validating webhook configuration")
 		}()
+		// ensure backend is ready before proceeding
+		waitWebhookConfigurationReady(f)
 		ginkgo.By("Creating a configMap that does not comply to the validation webhook rules")
 		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
 			cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f)
@@ -509,6 +512,7 @@ var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() {
 			},
 			Webhooks: []admissionregistrationv1.MutatingWebhook{
 				newMutateConfigMapWebhookFixture(f, context, 1, servicePort),
+				newMutatingIsReadyWebhookFixture(f, context, servicePort),
 			},
 		})
 		framework.ExpectNoError(err, "Creating mutating webhook configuration")
@@ -516,7 +520,8 @@ var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() {
 			err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(hook.Name, nil)
 			framework.ExpectNoError(err, "Deleting mutating webhook configuration")
 		}()
-
+		// ensure backend is ready before proceeding
+		waitWebhookConfigurationReady(f)
 		hook, err = admissionClient.MutatingWebhookConfigurations().Get(f.UniqueName, metav1.GetOptions{})
 		framework.ExpectNoError(err, "Getting mutating webhook configuration")
 		ginkgo.By("Updating a mutating webhook configuration's rules to not include the create operation")
@@ -778,6 +783,18 @@ func deployWebhookAndService(f *framework.Framework, image string, context *cert
 				// Use a non-default port for containers.
 				fmt.Sprintf("--port=%d", containerPort),
 			},
+			ReadinessProbe: &v1.Probe{
+				Handler: v1.Handler{
+					HTTPGet: &v1.HTTPGetAction{
+						Scheme: v1.URISchemeHTTPS,
+						Port:   intstr.FromInt(int(containerPort)),
+						Path:   "/readyz",
+					},
+				},
+				PeriodSeconds:    1,
+				SuccessThreshold: 1,
+				FailureThreshold: 30,
+			},
 			Image: image,
 			Ports: []v1.ContainerPort{{ContainerPort: containerPort}},
 		},
@@ -2049,8 +2066,8 @@ func registerValidatingWebhookForCRD(f *framework.Framework, configName string,
 				},
 				SideEffects:             &sideEffectsNone,
 				AdmissionReviewVersions: []string{"v1", "v1beta1"},
-				// Scope the webhook to just this namespace
+				// Scope the webhook to just this test
-				NamespaceSelector: &metav1.LabelSelector{
+				ObjectSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{f.UniqueName: "true"},
 				},
 			},
@@ -2101,6 +2118,9 @@ func testCRDDenyWebhook(f *framework.Framework) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name + "s." + group,
 			Labels: map[string]string{
+				// this label ensures our object is routed to this test's webhook
+				f.UniqueName: "true",
+				// this is the label the webhook disallows
 				"webhook-e2e-test": "webhook-disallow",
 			},
 		},
@@ -2434,6 +2454,7 @@ func waitWebhookConfigurationReady(f *framework.Framework) error {
 // with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready.
 func newValidatingIsReadyWebhookFixture(f *framework.Framework, context *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook {
 	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
+	failOpen := admissionregistrationv1.Ignore
 	return admissionregistrationv1.ValidatingWebhook{
 		Name: "validating-is-webhook-configuration-ready.k8s.io",
 		Rules: []admissionregistrationv1.RuleWithOperations{{
@@ -2453,6 +2474,8 @@ func newValidatingIsReadyWebhookFixture(f *framework.Framework, context *certCon
 			},
 			CABundle: context.signingCert,
 		},
+		// network failures while the service network routing is being set up should be ignored by the marker
+		FailurePolicy:           &failOpen,
 		SideEffects:             &sideEffectsNone,
 		AdmissionReviewVersions: []string{"v1", "v1beta1"},
 		// Scope the webhook to just the markers namespace
@@ -2470,6 +2493,7 @@ func newValidatingIsReadyWebhookFixture(f *framework.Framework, context *certCon
 // with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready.
 func newMutatingIsReadyWebhookFixture(f *framework.Framework, context *certContext, servicePort int32) admissionregistrationv1.MutatingWebhook {
 	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
+	failOpen := admissionregistrationv1.Ignore
 	return admissionregistrationv1.MutatingWebhook{
 		Name: "mutating-is-webhook-configuration-ready.k8s.io",
 		Rules: []admissionregistrationv1.RuleWithOperations{{
@@ -2489,6 +2513,8 @@ func newMutatingIsReadyWebhookFixture(f *framework.Framework, context *certConte
 			},
 			CABundle: context.signingCert,
 		},
+		// network failures while the service network routing is being set up should be ignored by the marker
+		FailurePolicy:           &failOpen,
 		SideEffects:             &sideEffectsNone,
 		AdmissionReviewVersions: []string{"v1", "v1beta1"},
 		// Scope the webhook to just the markers namespace

test/utils/image/manifest.go

@@ -204,7 +204,7 @@ const (
 
 func initImageConfigs() map[int]Config {
 	configs := map[int]Config{}
-	configs[Agnhost] = Config{e2eRegistry, "agnhost", "2.5"}
+	configs[Agnhost] = Config{e2eRegistry, "agnhost", "2.6"}
 	configs[Alpine] = Config{dockerLibraryRegistry, "alpine", "3.7"}
 	configs[AuthenticatedAlpine] = Config{gcAuthenticatedRegistry, "alpine", "3.7"}
 	configs[AuthenticatedWindowsNanoServer] = Config{gcAuthenticatedRegistry, "windows-nanoserver", "v1"}