Merge pull request #77713 from zjj2wry/secret_resync_duration

add secret back to the workqueue with delay time, avoid expired bootstrap tokens not being deleted

pkg/controller/bootstrap/BUILD

@@ -48,6 +48,7 @@ go_test(
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/informers:go_default_library",
         "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",

pkg/controller/bootstrap/tokencleaner.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"time"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -188,7 +188,8 @@ func (tc *TokenCleaner) syncFunc(key string) error {
 
 func (tc *TokenCleaner) evalSecret(o interface{}) {
 	secret := o.(*v1.Secret)
-	if bootstrapsecretutil.HasExpired(secret, time.Now()) {
+	ttl, alreadyExpired := bootstrapsecretutil.GetExpiration(secret, time.Now())
+	if alreadyExpired {
 		klog.V(3).Infof("Deleting expired secret %s/%s", secret.Namespace, secret.Name)
 		var options *metav1.DeleteOptions
 		if len(secret.UID) > 0 {
@@ -200,5 +201,7 @@ func (tc *TokenCleaner) evalSecret(o interface{}) {
 		if err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) {
 			klog.V(3).Infof("Error deleting Secret: %v", err)
 		}
+	} else if ttl > 0 {
+		tc.queue.AddAfter(o, ttl)
 	}
 }

pkg/controller/bootstrap/tokencleaner_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes/fake"
@@ -100,3 +101,46 @@ func TestCleanerNotExpired(t *testing.T) {
 
 	verifyActions(t, expected, cl.Actions())
 }
+
+func TestCleanerExpiredAt(t *testing.T) {
+	cleaner, cl, secrets, err := newTokenCleaner()
+	if err != nil {
+		t.Fatalf("error creating TokenCleaner: %v", err)
+	}
+
+	secret := newTokenSecret("tokenID", "tokenSecret")
+	addSecretExpiration(secret, timeString(2*time.Second))
+	expected := []core.Action{}
+	verifyFunc := func() {
+		secrets.Informer().GetIndexer().Add(secret)
+		cleaner.evalSecret(secret)
+		verifyActions(t, expected, cl.Actions())
+	}
+	// token has not expired currently
+	verifyFunc()
+
+	if cleaner.queue.Len() != 0 {
+		t.Errorf("not using the queue, the length should be 0, now: %v", cleaner.queue.Len())
+	}
+
+	var conditionFunc = func() (bool, error) {
+		if cleaner.queue.Len() == 1 {
+			return true, nil
+		}
+		return false, nil
+	}
+
+	err = wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, conditionFunc)
+	if err != nil {
+		t.Fatalf("secret is put back into the queue, the queue length should be 1, error: %v\n", err)
+	}
+
+	// secret was eventually deleted
+	expected = []core.Action{
+		core.NewDeleteAction(
+			schema.GroupVersionResource{Version: "v1", Resource: "secrets"},
+			api.NamespaceSystem,
+			secret.ObjectMeta.Name),
+	}
+	verifyFunc()
+}

staging/src/k8s.io/cluster-bootstrap/util/secrets/secrets.go

@@ -21,7 +21,7 @@ import (
 	"strings"
 	"time"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/cluster-bootstrap/token/api"
 	legacyutil "k8s.io/cluster-bootstrap/token/util"
@@ -46,21 +46,35 @@ func GetData(secret *v1.Secret, key string) string {
 
 // HasExpired will identify whether the secret expires
 func HasExpired(secret *v1.Secret, currentTime time.Time) bool {
+	_, expired := GetExpiration(secret, currentTime)
+
+	return expired
+}
+
+// GetExpiration checks if the secret expires
+// isExpired indicates if the secret is already expired.
+// timeRemaining indicates how long until it does expire.
+// if the secret has no expiration timestamp, returns 0, false.
+// if there is an error parsing the secret's expiration timestamp, returns 0, true.
+func GetExpiration(secret *v1.Secret, currentTime time.Time) (timeRemaining time.Duration, isExpired bool) {
 	expiration := GetData(secret, api.BootstrapTokenExpirationKey)
-	if len(expiration) > 0 {
-		expTime, err2 := time.Parse(time.RFC3339, expiration)
-		if err2 != nil {
-			klog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.",
-				expiration, secret.Namespace, secret.Name, err2)
-			return true
-		}
-		if currentTime.After(expTime) {
-			klog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v",
-				secret.Namespace, secret.Name, expiration)
-			return true
-		}
+	if len(expiration) == 0 {
+		return 0, false
 	}
-	return false
+	expTime, err := time.Parse(time.RFC3339, expiration)
+	if err != nil {
+		klog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.",
+			expiration, secret.Namespace, secret.Name, err)
+		return 0, true
+	}
+
+	timeRemaining = expTime.Sub(currentTime)
+	if timeRemaining <= 0 {
+		klog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v",
+			secret.Namespace, secret.Name, expiration)
+		return 0, true
+	}
+	return timeRemaining, false
 }
 
 // ParseName parses the name of the secret to extract the secret ID.