diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index 2ee7f9afc54..46eb6269d9d 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -91,6 +91,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { // Make sure the namespace created for the test is labeled to be selected by the webhooks labelNamespace(f, f.Namespace.Name) + createWebhookConfigurationReadyNamespace(f) ginkgo.By("Setting up server cert") context = setupServerCert(namespaceName, serviceName) @@ -150,7 +151,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { defer validatingWebhookCleanup() mutatingWebhookCleanup := registerMutatingWebhookForWebhookConfigurations(f, f.UniqueName+"blocking", context, servicePort) defer mutatingWebhookCleanup() - testWebhooksForWebhookConfigurations(f, f.UniqueName, servicePort) + testWebhooksForWebhookConfigurations(f, f.UniqueName, context, servicePort) }) ginkgo.It("Should mutate custom resource", func() { @@ -683,12 +684,15 @@ func registerWebhook(f *framework.Framework, configName string, context *certCon // Server cannot talk to this webhook, so it always fails. // Because this webhook is configured fail-open, request should be admitted after the call fails. failOpenHook, + + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil) @@ -733,12 +737,14 @@ func registerWebhookForAttachingPod(f *framework.Framework, configName string, c MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil) @@ -758,12 +764,14 @@ func registerMutatingWebhookForConfigMap(f *framework.Framework, configName stri Webhooks: []admissionregistrationv1.MutatingWebhook{ newMutateConfigMapWebhookFixture(f, context, 1, servicePort), newMutateConfigMapWebhookFixture(f, context, 2, servicePort), + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newMutatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) } } @@ -821,12 +829,14 @@ func registerMutatingWebhookForPod(f *framework.Framework, configName string, co MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newMutatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) } } @@ -1067,12 +1077,14 @@ func registerFailClosedWebhook(f *framework.Framework, configName string, contex // Server cannot talk to this webhook, so it always fails. // Because this webhook is configured fail-closed, request should be rejected after the call fails. hook, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { f.ClientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil) } @@ -1151,12 +1163,14 @@ func registerValidatingWebhookForWebhookConfigurations(f *framework.Framework, c MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil) framework.ExpectNoError(err, "deleting webhook config %s with namespace %s", configName, namespace) @@ -1210,12 +1224,14 @@ func registerMutatingWebhookForWebhookConfigurations(f *framework.Framework, con MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newMutatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) framework.ExpectNoError(err, "deleting webhook config %s with namespace %s", configName, namespace) @@ -1225,7 +1241,7 @@ func registerMutatingWebhookForWebhookConfigurations(f *framework.Framework, con // This test assumes that the deletion-rejecting webhook defined in // registerValidatingWebhookForWebhookConfigurations and the webhook-config-mutating // webhook defined in registerMutatingWebhookForWebhookConfigurations already exist. -func testWebhooksForWebhookConfigurations(f *framework.Framework, configName string, servicePort int32) { +func testWebhooksForWebhookConfigurations(f *framework.Framework, configName string, context *certContext, servicePort int32) { var err error client := f.ClientSet ginkgo.By("Creating a dummy validating-webhook-configuration object") @@ -1271,6 +1287,8 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) @@ -1278,8 +1296,8 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str e2elog.Failf("expected %s not to be mutated by mutating webhooks but it was", configName) } - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") ginkgo.By("Deleting the validating-webhook-configuration, which should be possible to remove") @@ -1325,6 +1343,8 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newMutatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) @@ -1332,8 +1352,8 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str e2elog.Failf("expected %s not to be mutated by mutating webhooks but it was", configName) } - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") ginkgo.By("Deleting the mutating-webhook-configuration, which should be possible to remove") @@ -1542,12 +1562,14 @@ func registerWebhookForCustomResource(f *framework.Framework, configName string, MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering custom resource webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil) } @@ -1617,12 +1639,14 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, configName MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newMutatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering custom resource webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) } } @@ -1819,12 +1843,14 @@ func registerValidatingWebhookForCRD(f *framework.Framework, configName string, MatchLabels: map[string]string{f.UniqueName: "true"}, }, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering crd webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil) } @@ -1943,12 +1969,14 @@ func registerSlowWebhook(f *framework.Framework, configName string, context *cer SideEffects: &sideEffectsNone, AdmissionReviewVersions: []string{"v1", "v1beta1"}, }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newValidatingIsReadyWebhookFixture(f, context, servicePort), }, }) framework.ExpectNoError(err, "registering slow webhook config %s with namespace %s", configName, namespace) - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") return func() { client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil) @@ -2148,3 +2176,117 @@ func newMutateConfigMapWebhookFixture(f *framework.Framework, context *certConte }, } } + +// createWebhookConfigurationReadyNamespace creates a separate namespace for webhook configuration ready markers to +// prevent cross-talk with webhook configurations being tested. +func createWebhookConfigurationReadyNamespace(f *framework.Framework) { + ns, err := f.ClientSet.CoreV1().Namespaces().Create(&v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.Namespace.Name + "-markers", + Labels: map[string]string{f.UniqueName + "-markers": "true"}, + }, + }) + framework.ExpectNoError(err, "creating namespace for webhook configuration ready markers") + f.AddNamespacesToDelete(ns) +} + +// waitWebhookConfigurationReady sends "marker" requests until a webhook configuration is ready. +// A webhook created with newValidatingIsReadyWebhookFixture or newMutatingIsReadyWebhookFixture should first be added to +// the webhook configuration. +func waitWebhookConfigurationReady(f *framework.Framework) error { + cmClient := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name + "-markers") + return wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { + marker := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: string(uuid.NewUUID()), + Labels: map[string]string{ + f.UniqueName: "true", + }, + }, + } + _, err := cmClient.Create(marker) + if err != nil { + // The always-deny webhook does not provide a reason, so check for the error string we expect + if strings.Contains(err.Error(), "denied") { + return true, nil + } + return false, err + } + // best effort cleanup of markers that are no longer needed + _ = cmClient.Delete(marker.GetName(), nil) + framework.Logf("Waiting for webhook configuration to be ready...") + return false, nil + }) +} + +// newValidatingIsReadyWebhookFixture creates a validating webhook that can be added to a webhook configuration and then probed +// with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready. +func newValidatingIsReadyWebhookFixture(f *framework.Framework, context *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook { + sideEffectsNone := admissionregistrationv1.SideEffectClassNone + return admissionregistrationv1.ValidatingWebhook{ + Name: "validating-is-webhook-configuration-ready.k8s.io", + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"configmaps"}, + }, + }}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceName, + Path: strPtr("/always-deny"), + Port: pointer.Int32Ptr(servicePort), + }, + CABundle: context.signingCert, + }, + SideEffects: &sideEffectsNone, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, + // Scope the webhook to just the markers namespace + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{f.UniqueName + "-markers": "true"}, + }, + // appease createValidatingWebhookConfiguration isolation requirements + ObjectSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{f.UniqueName: "true"}, + }, + } +} + +// newMutatingIsReadyWebhookFixture creates a mutating webhook that can be added to a webhook configuration and then probed +// with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready. +func newMutatingIsReadyWebhookFixture(f *framework.Framework, context *certContext, servicePort int32) admissionregistrationv1.MutatingWebhook { + sideEffectsNone := admissionregistrationv1.SideEffectClassNone + return admissionregistrationv1.MutatingWebhook{ + Name: "mutating-is-webhook-configuration-ready.k8s.io", + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"configmaps"}, + }, + }}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceName, + Path: strPtr("/always-deny"), + Port: pointer.Int32Ptr(servicePort), + }, + CABundle: context.signingCert, + }, + SideEffects: &sideEffectsNone, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, + // Scope the webhook to just the markers namespace + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{f.UniqueName + "-markers": "true"}, + }, + // appease createMutatingWebhookConfiguration isolation requirements + ObjectSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{f.UniqueName: "true"}, + }, + } +}