From 1608fc5e8825bb13b1ca391253571dfcad18eac1 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Tue, 25 Jan 2022 13:11:52 -0500 Subject: [PATCH 1/2] e2e: Wait for kube-root-ca.crt to be created The change from service account secrets to projected tokens and the new dependency on kube-root-ca.crt to start pods with those projected tokens means that e2e tests can start before kube-root-ca.crt is created in a namespace. Wait for the default service account AND the kube-root-ca.crt configmap in normal e2e tests. --- test/e2e/framework/framework.go | 3 +++ test/e2e/framework/util.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 5d72e89f792..6472b89195a 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -248,6 +248,9 @@ func (f *Framework) BeforeEach() { ginkgo.By("Waiting for a default service account to be provisioned in namespace") err = WaitForDefaultServiceAccountInNamespace(f.ClientSet, namespace.Name) ExpectNoError(err) + ginkgo.By("Waiting for kube-root-ca.crt to be provisioned in namespace") + err = WaitForKubeRootCAInNamespace(f.ClientSet, namespace.Name) + ExpectNoError(err) } else { Logf("Skipping waiting for service account") } diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 3f3602e9f3f..8d4ee0754bc 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -282,6 +282,32 @@ func WaitForNamespacesDeleted(c clientset.Interface, namespaces []string, timeou }) } +func waitForConfigMapInNamespace(c clientset.Interface, ns, name string, timeout time.Duration) error { + fieldSelector := fields.OneTermEqualSelector("metadata.name", name).String() + lw := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) { + options.FieldSelector = fieldSelector + return c.CoreV1().ConfigMaps(ns).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) { + options.FieldSelector = fieldSelector + return c.CoreV1().ConfigMaps(ns).Watch(context.TODO(), options) + }, + } + ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout) + defer cancel() + _, err := watchtools.UntilWithSync(ctx, lw, &v1.ConfigMap{}, nil, func(event watch.Event) (bool, error) { + switch event.Type { + case watch.Deleted: + return false, apierrors.NewNotFound(schema.GroupResource{Resource: "configmaps"}, name) + case watch.Added, watch.Modified: + return true, nil + } + return false, nil + }) + return err +} + func waitForServiceAccountInNamespace(c clientset.Interface, ns, serviceAccountName string, timeout time.Duration) error { fieldSelector := fields.OneTermEqualSelector("metadata.name", serviceAccountName).String() lw := &cache.ListWatch{ @@ -321,6 +347,13 @@ func WaitForDefaultServiceAccountInNamespace(c clientset.Interface, namespace st return waitForServiceAccountInNamespace(c, namespace, "default", ServiceAccountProvisionTimeout) } +// WaitForKubeRootCAInNamespace waits for the configmap kube-root-ca.crt containing the service account +// CA trust bundle to be provisioned in the specified namespace so that pods do not have to retry mounting +// the config map (which creates noise that hides other issues in the Kubelet). +func WaitForKubeRootCAInNamespace(c clientset.Interface, namespace string) error { + return waitForConfigMapInNamespace(c, namespace, "kube-root-ca.crt", ServiceAccountProvisionTimeout) +} + // CreateTestingNS should be used by every test, note that we append a common prefix to the provided test name. // Please see NewFramework instead of using this directly. func CreateTestingNS(baseName string, c clientset.Interface, labels map[string]string) (*v1.Namespace, error) { From b4aa9a189ead272a3f6dd62562f7a355932c57fa Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Tue, 25 Jan 2022 13:17:03 -0500 Subject: [PATCH 2/2] e2e: Wait only for the service account Now that projected service account tokens do not require the secret to be created, exclude the wait condition on the token and simply wait for the service account. --- test/e2e/framework/util.go | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 8d4ee0754bc..137dcaa8cb3 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -322,24 +322,18 @@ func waitForServiceAccountInNamespace(c clientset.Interface, ns, serviceAccountN } ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout) defer cancel() - _, err := watchtools.UntilWithSync(ctx, lw, &v1.ServiceAccount{}, nil, serviceAccountHasSecrets) + _, err := watchtools.UntilWithSync(ctx, lw, &v1.ServiceAccount{}, nil, func(event watch.Event) (bool, error) { + switch event.Type { + case watch.Deleted: + return false, apierrors.NewNotFound(schema.GroupResource{Resource: "serviceaccounts"}, serviceAccountName) + case watch.Added, watch.Modified: + return true, nil + } + return false, nil + }) return err } -// serviceAccountHasSecrets returns true if the service account has at least one secret, -// false if it does not, or an error. -func serviceAccountHasSecrets(event watch.Event) (bool, error) { - switch event.Type { - case watch.Deleted: - return false, apierrors.NewNotFound(schema.GroupResource{Resource: "serviceaccounts"}, "") - } - switch t := event.Object.(type) { - case *v1.ServiceAccount: - return len(t.Secrets) > 0, nil - } - return false, nil -} - // WaitForDefaultServiceAccountInNamespace waits for the default service account to be provisioned // the default service account is what is associated with pods when they do not specify a service account // as a result, pods are not able to be provisioned in a namespace until the service account is provisioned