From dc0c21c7d7b264db2afb95db681505e341eb00ae Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Sun, 20 Oct 2019 16:09:19 -0400 Subject: [PATCH] serviceaccount: If namespace is terminating, ignore create errors In some scenarios the service account and token controllers can race with namespace deletion, causing a burst of errors as they attempt to recreate secrets being deleted. Instead, detect these errors and do not retry. --- pkg/controller/serviceaccount/serviceaccounts_controller.go | 5 ++++- pkg/controller/serviceaccount/tokens_controller.go | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/controller/serviceaccount/serviceaccounts_controller.go b/pkg/controller/serviceaccount/serviceaccounts_controller.go index c775706a605..83214feb9df 100644 --- a/pkg/controller/serviceaccount/serviceaccounts_controller.go +++ b/pkg/controller/serviceaccount/serviceaccounts_controller.go @@ -213,7 +213,10 @@ func (c *ServiceAccountsController) syncNamespace(key string) error { sa.Namespace = ns.Name if _, err := c.client.CoreV1().ServiceAccounts(ns.Name).Create(&sa); err != nil && !apierrs.IsAlreadyExists(err) { - createFailures = append(createFailures, err) + // we can safely ignore terminating namespace errors + if !apierrs.HasStatusCause(err, v1.NamespaceTerminatingCause) { + createFailures = append(createFailures, err) + } } } diff --git a/pkg/controller/serviceaccount/tokens_controller.go b/pkg/controller/serviceaccount/tokens_controller.go index 6e128a8da03..62dab048695 100644 --- a/pkg/controller/serviceaccount/tokens_controller.go +++ b/pkg/controller/serviceaccount/tokens_controller.go @@ -408,6 +408,10 @@ func (e *TokensController) ensureReferencedToken(serviceAccount *v1.ServiceAccou // Save the secret createdToken, err := e.client.CoreV1().Secrets(serviceAccount.Namespace).Create(secret) if err != nil { + // if the namespace is being terminated, create will fail no matter what + if apierrors.HasStatusCause(err, v1.NamespaceTerminatingCause) { + return false, err + } // retriable error return true, err }