Merge pull request #120682 from yt2985/cleanSA

LegacyServiceAccountTokenCleanUp beta

pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner.go

@@ -18,6 +18,7 @@ package serviceaccount
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -29,6 +30,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	applyv1 "k8s.io/client-go/applyconfigurations/core/v1"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	listersv1 "k8s.io/client-go/listers/core/v1"
@@ -183,7 +185,28 @@ func (tc *LegacySATokenCleaner) evaluateSATokens(ctx context.Context) {
 			continue
 		}
 
-		logger.Info("Delete auto-generated service account token", "secret", klog.KRef(secret.Namespace, secret.Name), "creationTime", secret.CreationTimestamp, "lastUsed", lastUsed)
+		invalidSince := secret.Labels[serviceaccount.InvalidSinceLabelKey]
+		// If the secret has not been labeled with invalid since date or the label value has invalid format, update the invalidSince label with the current date value.
+		_, err = time.Parse(dateFormat, invalidSince)
+		if err != nil {
+			invalidSince = now.Format(dateFormat)
+			logger.Info("Mark the auto-generated service account token as invalid", "invalidSince", invalidSince, "secret", klog.KRef(secret.Namespace, secret.Name))
+			patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithUID(secret.UID).WithLabels(map[string]string{serviceaccount.InvalidSinceLabelKey: invalidSince}))
+			if err != nil {
+				logger.Error(err, "Failed to marshal invalid since label")
+			} else {
+				if _, err := tc.client.CoreV1().Secrets(secret.Namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
+					logger.Error(err, "Failed to label legacy service account token secret with invalid since date")
+				}
+			}
+			continue
+		}
+
+		if invalidSince >= preserveUsedOnOrAfter {
+			continue
+		}
+
+		logger.Info("Delete auto-generated service account token", "secret", klog.KRef(secret.Namespace, secret.Name), "creationTime", secret.CreationTimestamp, "lastUsed", lastUsed, "invalidSince", invalidSince)
 		if err := tc.client.CoreV1().Secrets(secret.Namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &secret.ResourceVersion}}); err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) {
 			logger.Error(err, "Deleting legacy service account token", "secret", klog.KRef(secret.Namespace, secret.Name), "serviceaccount", sa.Name)
 		}

pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner_test.go

@@ -18,6 +18,7 @@ package serviceaccount
 
 import (
 	"context"
+	"encoding/json"
 	"reflect"
 	"testing"
 	"time"
@@ -26,6 +27,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	applyv1 "k8s.io/client-go/applyconfigurations/core/v1"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
@@ -47,7 +50,7 @@ func configuredConfigMap(label string) *v1.ConfigMap {
 	}
 }
 
-func configuredServiceAccountTokenSecret(label, creationTimeString, serviceAccountName, serviceAccountUID, deletionTimeString string) *v1.Secret {
+func configuredServiceAccountTokenSecret(lastUsedLabel, invalidSinceLabel, creationTimeString, serviceAccountName, serviceAccountUID, deletionTimeString string) *v1.Secret {
 	var deletionTime *metav1.Time
 	if deletionTimeString == "" {
 		deletionTime = nil
@@ -56,8 +59,11 @@ func configuredServiceAccountTokenSecret(label, creationTimeString, serviceAccou
 	}
 	creationTime, _ := time.Parse(dateFormat, creationTimeString)
 	labels := map[string]string{}
-	if label != "" {
+	if lastUsedLabel != "" {
-		labels[serviceaccount.LastUsedLabelKey] = label
+		labels[serviceaccount.LastUsedLabelKey] = lastUsedLabel
+	}
+	if invalidSinceLabel != "" {
+		labels[serviceaccount.InvalidSinceLabelKey] = invalidSinceLabel
 	}
 	return &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
@@ -108,6 +114,11 @@ func configuredPod(withSecretMount bool) *v1.Pod {
 	}
 }
 
+func patchContent(namespace, name, invalidSince string, uID types.UID) []byte {
+	patch, _ := json.Marshal(applyv1.Secret(name, namespace).WithUID(uID).WithLabels(map[string]string{serviceaccount.InvalidSinceLabelKey: invalidSince}))
+	return patch
+}
+
 func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 	testcases := map[string]struct {
 		LegacyTokenCleanUpPeriod time.Duration
@@ -120,7 +131,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 		ExpectedActions []core.Action
 	}{
 		"configmap does not exist": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-28"),
@@ -129,7 +140,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"configmap exists, but the configmap does not have tracked-since label": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("")},
@@ -139,7 +150,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"configmap exists, the time period since 'tracked-since' is smaller than the CleanUpPeriod": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-29")},
@@ -149,7 +160,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"configmap exists, the 'tracked-since' cannot be parsed": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27-1")},
@@ -169,7 +180,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"secret is not referenced by serviceaccount": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(emptySecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27")},
@@ -179,7 +190,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"auto-generated secret has a late creation time": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-30", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-30", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27")},
@@ -189,7 +200,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"auto-generated secret has a deletion time": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", "deleted"),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", "deleted"),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27")},
@@ -199,7 +210,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"auto-generated secret has a late last-used time": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-30", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-30", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27")},
@@ -209,7 +220,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"auto-generated secret has a last-used label, but it can not be parsed": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27-1", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27-1", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27")},
@@ -219,7 +230,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"secret-referenced service account does not exist": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", ""),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27")},
 			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"),
@@ -228,7 +239,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"secret-referenced service account uid does not match": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "123456", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "123456", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27")},
@@ -238,7 +249,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			},
 		},
 		"secret-referenced service account name is empty": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-27")},
@@ -247,24 +258,66 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
 			},
 		},
-		"auto-generated secret does not have 'last-used' label": {
+		"auto-generated secret does not have 'last-used' label, has not been marked as invalid": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-28")},
 			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-30"),
+			ExpectedActions: []core.Action{
+				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
+				core.NewPatchAction(
+					schema.GroupVersionResource{Version: "v1", Resource: "secrets"},
+					metav1.NamespaceDefault, "token-secret-1",
+					types.MergePatchType,
+					patchContent(metav1.NamespaceDefault, "token-secret-1", time.Now().UTC().Format(dateFormat), types.UID("23456")),
+				),
+			},
+		},
+		"auto-generated secret does not have 'last-used' label, has been marked as invalid, invalid_since label can not be parsed": {
+			ExistingSecret:           configuredServiceAccountTokenSecret("", "2022-12-29-1", "2022-12-27", "default", "12345", ""),
+			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
+			ExistingPod:              configuredPod(false),
+			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-28")},
+			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-30"),
+			ExpectedActions: []core.Action{
+				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
+				core.NewPatchAction(
+					schema.GroupVersionResource{Version: "v1", Resource: "secrets"},
+					metav1.NamespaceDefault, "token-secret-1",
+					types.MergePatchType,
+					patchContent(metav1.NamespaceDefault, "token-secret-1", time.Now().UTC().Format(dateFormat), types.UID("23456")),
+				),
+			},
+		},
+		"auto-generated secret does not have 'last-used' label, has been marked as invalid, time period since invalid is less than CleanUpPeriod": {
+			ExistingSecret:           configuredServiceAccountTokenSecret("", "2023-01-01", "2022-12-27", "default", "12345", ""),
+			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
+			ExistingPod:              configuredPod(false),
+			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-28")},
+			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"),
+			ExpectedActions: []core.Action{
+				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
+			},
+		},
+		"auto-generated secret does not have 'last-used' label, has been marked as invalid, time period since invalid is larger than CleanUpPeriod": {
+			ExistingSecret:           configuredServiceAccountTokenSecret("", "2022-12-29", "2022-12-27", "default", "12345", ""),
+			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
+			ExistingPod:              configuredPod(false),
+			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-28")},
+			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2023-01-01"),
 			ExpectedActions: []core.Action{
 				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
 				core.NewDeleteActionWithOptions(
 					schema.GroupVersionResource{Version: "v1", Resource: "secrets"},
 					metav1.NamespaceDefault, "token-secret-1",
 					metav1.DeleteOptions{
-						Preconditions: &metav1.Preconditions{ResourceVersion: &configuredServiceAccountTokenSecret("", "2022-12-27", "default", "12345", "").ResourceVersion},
+						Preconditions: &metav1.Preconditions{ResourceVersion: &configuredServiceAccountTokenSecret("", "2022-12-29", "2022-12-27", "default", "12345", "").ResourceVersion},
 					}),
 			},
 		},
 		"auto-generated secret is mounted by the pod": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(true),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-28")},
@@ -273,20 +326,45 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
 			},
 		},
-		"auto-generated secret has 'last-used' label, the time period since last-used is larger than CleanUpPeriod": {
+		"auto-generated secret has 'last-used' label, the time period since last-used is larger than CleanUpPeriod, secret has not been marked as invalid": {
-			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""),
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "", "2022-12-27", "default", "12345", ""),
 			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
 			ExistingPod:              configuredPod(false),
 			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-28")},
 			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-30"),
+			ExpectedActions: []core.Action{
+				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
+				core.NewPatchAction(
+					schema.GroupVersionResource{Version: "v1", Resource: "secrets"},
+					metav1.NamespaceDefault, "token-secret-1",
+					types.MergePatchType,
+					patchContent(metav1.NamespaceDefault, "token-secret-1", time.Now().UTC().Format(dateFormat), types.UID("23456")),
+				),
+			},
+		},
+		"auto-generated secret has 'last-used' label, the time period since last-used is larger than CleanUpPeriod, secret has been marked as invalid, time peroid since invalid is less than CleanUpPeriod": {
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2023-05-01", "2022-12-27", "default", "12345", ""),
+			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
+			ExistingPod:              configuredPod(false),
+			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-28")},
+			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-30"),
+			ExpectedActions: []core.Action{
+				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
+			},
+		},
+		"auto-generated secret has 'last-used' label, the time period since last-used is larger than CleanUpPeriod, secret has been marked as invalid, time peroid since invalid is larger than CleanUpPeriod": {
+			ExistingSecret:           configuredServiceAccountTokenSecret("2022-12-27", "2023-01-05", "2022-12-27", "default", "12345", ""),
+			ExistingServiceAccount:   serviceAccount(tokenSecretReferences()),
+			ExistingPod:              configuredPod(false),
+			ClientObjects:            []runtime.Object{configuredConfigMap("2022-12-28")},
+			LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2023-05-01"),
 			ExpectedActions: []core.Action{
 				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName),
 				core.NewDeleteActionWithOptions(
 					schema.GroupVersionResource{Version: "v1", Resource: "secrets"},
 					metav1.NamespaceDefault, "token-secret-1",
-					metav1.DeleteOptions{Preconditions: &metav1.Preconditions{
+					metav1.DeleteOptions{
-						ResourceVersion: &configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", "").ResourceVersion,
+						Preconditions: &metav1.Preconditions{ResourceVersion: &configuredServiceAccountTokenSecret("", "2023-01-05", "2022-12-27", "default", "12345", "").ResourceVersion},
-					},
 					}),
 			},
 		},

pkg/features/kube_features.go

@@ -475,6 +475,7 @@ const (
 	// owner: @yt2985
 	// kep: http://kep.k8s.io/2800
 	// alpha: v1.28
+	// beta: v1.29
 	//
 	// Enables cleaning up of secret-based service account tokens.
 	LegacyServiceAccountTokenCleanUp featuregate.Feature = "LegacyServiceAccountTokenCleanUp"
@@ -1014,7 +1015,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	LegacyServiceAccountTokenTracking: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.30
 
-	LegacyServiceAccountTokenCleanUp: {Default: false, PreRelease: featuregate.Alpha},
+	LegacyServiceAccountTokenCleanUp: {Default: true, PreRelease: featuregate.Beta},
 
 	LocalStorageCapacityIsolationFSQuotaMonitoring: {Default: false, PreRelease: featuregate.Alpha},
 

pkg/serviceaccount/jwt_test.go

@@ -151,6 +151,15 @@ func TestTokenGenerateAndValidate(t *testing.T) {
 			Namespace: "test",
 		},
 	}
+	invalidAutoSecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-rsa-secret",
+			Namespace: "test",
+			Labels: map[string]string{
+				"kubernetes.io/legacy-token-invalid-since": "2022-12-20",
+			},
+		},
+	}
 	ecdsaSecret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "my-ecdsa-secret",
@@ -176,6 +185,20 @@ func TestTokenGenerateAndValidate(t *testing.T) {
 
 	checkJSONWebSignatureHasKeyID(t, rsaToken, rsaKeyID)
 
+	// Generate RSA token with invalidAutoSecret
+	invalidAutoSecretToken, err := rsaGenerator.GenerateToken(serviceaccount.LegacyClaims(*serviceAccount, *invalidAutoSecret))
+	if err != nil {
+		t.Fatalf("error generating token: %v", err)
+	}
+	if len(invalidAutoSecretToken) == 0 {
+		t.Fatalf("no token generated")
+	}
+	invalidAutoSecret.Data = map[string][]byte{
+		"token": []byte(invalidAutoSecretToken),
+	}
+
+	checkJSONWebSignatureHasKeyID(t, invalidAutoSecretToken, rsaKeyID)
+
 	// Generate the ECDSA token
 	ecdsaGenerator, err := serviceaccount.JWTTokenGenerator(serviceaccount.LegacyIssuer, getPrivateKey(ecdsaPrivateKey))
 	if err != nil {
@@ -327,6 +350,12 @@ func TestTokenGenerateAndValidate(t *testing.T) {
 			ExpectedErr: true,
 			ExpectedOK:  false,
 		},
+		"secret is marked as invalid": {
+			Token:       invalidAutoSecretToken,
+			Client:      fake.NewSimpleClientset(serviceAccount, invalidAutoSecret),
+			Keys:        []interface{}{getPublicKey(rsaPublicKey)},
+			ExpectedErr: true,
+		},
 	}
 
 	for k, tc := range testCases {

pkg/serviceaccount/legacy.go

@@ -36,6 +36,8 @@ import (
 	"k8s.io/klog/v2"
 )
 
+const InvalidSinceLabelKey = "kubernetes.io/legacy-token-invalid-since"
+
 func LegacyClaims(serviceAccount v1.ServiceAccount, secret v1.Secret) (*jwt.Claims, interface{}) {
 	return &jwt.Claims{
 			Subject: apiserverserviceaccount.MakeUsername(serviceAccount.Namespace, serviceAccount.Name),
@@ -148,6 +150,14 @@ func (v *legacyValidator) Validate(ctx context.Context, tokenData string, public
 		// Track secret-based long-lived service account tokens and add audit annotations and metrics.
 		autoGenerated := false
 
+		// Check if the secret has been marked as invalid
+		if invalidSince := secret.Labels[InvalidSinceLabelKey]; invalidSince != "" {
+			audit.AddAuditAnnotation(ctx, "authentication.k8s.io/legacy-token-invalidated", secret.Name+"/"+secret.Namespace)
+			invalidatedAutoTokensTotal.WithContext(ctx).Inc()
+			v.patchSecretWithLastUsedDate(ctx, secret)
+			return nil, fmt.Errorf("the token in secret %s/%s for service account %s/%s has been marked invalid. Use tokens from the TokenRequest API or manually created secret-based tokens, or remove the '%s' label from the secret to temporarily allow use of this token", namespace, secretName, namespace, serviceAccountName, InvalidSinceLabelKey)
+		}
+
 		// Check if it is an auto-generated secret-based token
 		for _, ref := range serviceAccount.Secrets {
 			if ref.Name == secret.Name {
@@ -165,20 +175,7 @@ func (v *legacyValidator) Validate(ctx context.Context, tokenData string, public
 			manuallyCreatedTokensTotal.WithContext(ctx).Inc()
 		}
 
-		now := time.Now().UTC()
+		v.patchSecretWithLastUsedDate(ctx, secret)
-		today := now.Format("2006-01-02")
-		tomorrow := now.AddDate(0, 0, 1).Format("2006-01-02")
-		lastUsed := secret.Labels[LastUsedLabelKey]
-		if lastUsed != today && lastUsed != tomorrow {
-			patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithLabels(map[string]string{LastUsedLabelKey: today}))
-			if err != nil {
-				klog.Errorf("Failed to marshal legacy service account token tracking labels, err: %v", err)
-			} else {
-				if _, err := v.secretsWriter.Secrets(namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
-					klog.Errorf("Failed to label legacy service account token secret with last-used, err: %v", err)
-				}
-			}
-		}
 	}
 
 	return &apiserverserviceaccount.ServiceAccountInfo{
@@ -188,6 +185,23 @@ func (v *legacyValidator) Validate(ctx context.Context, tokenData string, public
 	}, nil
 }
 
+func (v *legacyValidator) patchSecretWithLastUsedDate(ctx context.Context, secret *v1.Secret) {
+	now := time.Now().UTC()
+	today := now.Format("2006-01-02")
+	tomorrow := now.AddDate(0, 0, 1).Format("2006-01-02")
+	lastUsed := secret.Labels[LastUsedLabelKey]
+	if lastUsed != today && lastUsed != tomorrow {
+		patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithUID(secret.UID).WithLabels(map[string]string{LastUsedLabelKey: today}))
+		if err != nil {
+			klog.Errorf("Failed to marshal legacy service account token %s/%s tracking labels, err: %w", secret.Name, secret.Namespace, err)
+		} else {
+			if _, err := v.secretsWriter.Secrets(secret.Namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
+				klog.Errorf("Failed to label legacy service account token %s/%s with last-used date, err: %w", secret.Name, secret.Namespace, err)
+			}
+		}
+	}
+}
+
 func (v *legacyValidator) NewPrivateClaims() interface{} {
 	return &legacyPrivateClaims{}
 }

pkg/serviceaccount/metrics.go

@@ -67,6 +67,16 @@ var (
 		},
 	)
 
+	// invalidatedAutoTokensTotal is the number of uses of automatically generated secret-based long lived tokens that have been marked as invalid.
+	invalidatedAutoTokensTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Subsystem:      kubeServiceAccountSubsystem,
+			Name:           "invalid_legacy_auto_token_uses_total",
+			Help:           "Cumulative invalid auto-generated legacy tokens used",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+
 	// ValidTokensTotal is the number of valid projected tokens used.
 	validTokensTotal = metrics.NewCounter(
 		&metrics.CounterOpts{
@@ -86,6 +96,7 @@ func RegisterMetrics() {
 		legacyregistry.MustRegister(staleTokensTotal)
 		legacyregistry.MustRegister(manuallyCreatedTokensTotal)
 		legacyregistry.MustRegister(autoGeneratedTokensTotal)
+		legacyregistry.MustRegister(invalidatedAutoTokensTotal)
 		legacyregistry.MustRegister(validTokensTotal)
 	})
 }

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go

@@ -472,7 +472,7 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding)
 			ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "legacy-service-account-token-cleaner"},
 			Rules: []rbacv1.PolicyRule{
 				rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("configmaps").Names(legacytokentracking.ConfigMapName).RuleOrDie(),
-				rbacv1helpers.NewRule("delete").Groups(legacyGroup).Resources("secrets").RuleOrDie(),
+				rbacv1helpers.NewRule("patch", "delete").Groups(legacyGroup).Resources("secrets").RuleOrDie(),
 			},
 		})
 	}

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml

@@ -255,6 +255,23 @@ items:
   - kind: ServiceAccount
     name: job-controller
     namespace: kube-system
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    annotations:
+      rbac.authorization.kubernetes.io/autoupdate: "true"
+    creationTimestamp: null
+    labels:
+      kubernetes.io/bootstrapping: rbac-defaults
+    name: system:controller:legacy-service-account-token-cleaner
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:controller:legacy-service-account-token-cleaner
+  subjects:
+  - kind: ServiceAccount
+    name: legacy-service-account-token-cleaner
+    namespace: kube-system
 - apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRoleBinding
   metadata:

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml

@@ -830,6 +830,31 @@ items:
     - create
     - patch
     - update
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    annotations:
+      rbac.authorization.kubernetes.io/autoupdate: "true"
+    creationTimestamp: null
+    labels:
+      kubernetes.io/bootstrapping: rbac-defaults
+    name: system:controller:legacy-service-account-token-cleaner
+  rules:
+  - apiGroups:
+    - ""
+    resourceNames:
+    - kube-apiserver-legacy-service-account-token-tracking
+    resources:
+    - configmaps
+    verbs:
+    - get
+  - apiGroups:
+    - ""
+    resources:
+    - secrets
+    verbs:
+    - delete
+    - patch
 - apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRole
   metadata:

test/integration/serviceaccount/legacy_service_account_token_clean_up_test.go

@@ -20,6 +20,7 @@ package serviceaccount
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"testing"
 	"time"
@@ -27,8 +28,10 @@ import (
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	applyv1 "k8s.io/client-go/applyconfigurations/core/v1"
 	clientinformers "k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	listersv1 "k8s.io/client-go/listers/core/v1"
@@ -44,7 +47,9 @@ import (
 const (
 	dateFormat    = "2006-01-02"
 	cleanUpPeriod = 24 * time.Hour
-	syncInterval  = 1 * time.Second
+	syncInterval  = 5 * time.Second
+	pollTimeout   = 15 * time.Second
+	pollInterval  = time.Second
 )
 
 func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
@@ -57,20 +62,8 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 		t.Fatalf("failed to setup ServiceAccounts server: %v", err)
 	}
 
-	// wait configmap to label with tracking date
+	// wait configmap to be labeled with tracking date
-	if err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
+	waitConfigmapToBeLabeled(ctx, t, c)
-		configMap, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, legacytokentracking.ConfigMapName, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-		_, exist := configMap.Data[legacytokentracking.ConfigMapDataKey]
-		if !exist {
-			return false, fmt.Errorf("configMap does not have since label")
-		}
-		return true, nil
-	}); err != nil {
-		t.Fatalf("failed to wait configmap starts to track: %v", err)
-	}
 
 	tests := []struct {
 		name               string
@@ -78,6 +71,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 		secretTokenData    string
 		namespace          string
 		expectCleanedUp    bool
+		expectInvalidLabel bool
 		lastUsedLabel      bool
 		isPodMounted       bool
 		isManual           bool
@@ -90,6 +84,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			isManual:           false,
 			isPodMounted:       false,
 			expectCleanedUp:    true,
+			expectInvalidLabel: true,
 		},
 		{
 			name:               "manually created legacy token",
@@ -99,6 +94,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			isManual:           true,
 			isPodMounted:       false,
 			expectCleanedUp:    false,
+			expectInvalidLabel: false,
 		},
 		{
 			name:               "auto created legacy token with pod binding",
@@ -108,6 +104,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			isManual:           false,
 			isPodMounted:       true,
 			expectCleanedUp:    false,
+			expectInvalidLabel: false,
 		},
 		{
 			name:               "auto created legacy token without pod binding, secret has not been used after tracking",
@@ -117,6 +114,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			isManual:           false,
 			isPodMounted:       false,
 			expectCleanedUp:    true,
+			expectInvalidLabel: true,
 		},
 	}
 	for _, test := range tests {
@@ -152,10 +150,7 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			}
 			podLister := informers.Core().V1().Pods().Lister()
 			if test.isPodMounted {
-				_, err = createAutotokenMountedPod(c, test.namespace, test.secretName, podLister)
+				createAutotokenMountedPod(ctx, t, c, test.namespace, test.secretName, podLister)
-				if err != nil {
-					t.Fatalf("Pod not created: %v", err)
-				}
 			}
 
 			myConfig := *config
@@ -165,51 +160,191 @@ func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
 			roClient := clientset.NewForConfigOrDie(&myConfig)
 
 			// the secret should not be labeled with LastUsedLabelKey.
-			liveSecret, err := c.CoreV1().Secrets(test.namespace).Get(context.TODO(), test.secretName, metav1.GetOptions{})
+			checkLastUsedLabel(ctx, t, c, secret, false)
-			if err != nil {
+
-				t.Fatalf("Could not get secret: %v", err)
+			if test.lastUsedLabel {
-			}
+				doServiceAccountAPIReadRequest(ctx, t, roClient, test.namespace, true)
-			_, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey]
+
-			if ok {
+				// all service account tokens should be labeled with LastUsedLabelKey.
-				t.Fatalf("Secret %s should not have the lastUsed label", test.secretName)
+				checkLastUsedLabel(ctx, t, c, secret, true)
 			}
 
-			// authenticate legacy tokens
+			// Test invalid labels
-			if test.lastUsedLabel {
+			fakeClock.Step(cleanUpPeriod + 24*time.Hour)
-				doServiceAccountAPIRequests(t, roClient, test.namespace, true, true, false)
+			checkInvalidSinceLabel(ctx, t, c, secret, fakeClock, test.expectInvalidLabel)
-				// all service account tokens should be labeled with LastUsedLabelKey.
+
-				liveSecret, err = c.CoreV1().Secrets(test.namespace).Get(context.TODO(), test.secretName, metav1.GetOptions{})
+			// Test invalid secret cannot be used
-				if err != nil {
+			if test.expectInvalidLabel {
-					t.Fatalf("Could not get secret: %v", err)
+				t.Logf("Check the invalid token cannot authenticate request.")
-				}
+				doServiceAccountAPIReadRequest(ctx, t, roClient, test.namespace, false)
-				lastUsed, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey]
+
-				if !ok {
+				// Check the secret has been labelded with the LastUsedLabelKey.
-					t.Fatalf("The secret %s should be labeled lastUsed time: %s", test.secretName, lastUsed)
+				if !test.lastUsedLabel {
-				} else {
+					checkLastUsedLabel(ctx, t, c, secret, true)
-					t.Logf("The secret %s has been labeled with %s", test.secretName, lastUsed)
 				}
+
+				// Update secret by removing the invalid since label
+				removeInvalidLabel(ctx, c, t, secret)
+
+				t.Logf("Check the token can authenticate request after patching the secret by removing the invalid label.")
+				doServiceAccountAPIReadRequest(ctx, t, roClient, test.namespace, true)
+
+				// Update the lastUsed label date to the fakeClock date (as the validation function uses the real time to label the lastUsed date)
+				patchSecret(ctx, c, t, fakeClock.Now().UTC().Format(dateFormat), secret)
+
+				// The secret will be marked as invalid again after time period duration cleanUpPeriod + 24*time.Hour
+				fakeClock.Step(cleanUpPeriod + 24*time.Hour)
+				checkInvalidSinceLabel(ctx, t, c, secret, fakeClock, true)
 			}
 
 			fakeClock.Step(cleanUpPeriod + 24*time.Hour)
-			time.Sleep(2 * syncInterval)
+			checkSecretCleanUp(ctx, t, c, secret, test.expectCleanedUp)
-			liveSecret, err = c.CoreV1().Secrets(test.namespace).Get(context.TODO(), test.secretName, metav1.GetOptions{})
-			if test.expectCleanedUp {
-				if err == nil {
-					t.Fatalf("The secret %s should be cleaned up. time: %v; creationTime: %v", test.secretName, fakeClock.Now().UTC(), liveSecret.CreationTimestamp)
-				} else if !apierrors.IsNotFound(err) {
-					t.Fatalf("Failed to get secret %s, err: %v", test.secretName, err)
-				}
-			} else if err != nil {
-				if apierrors.IsNotFound(err) {
-					t.Fatalf("The secret %s should not be cleaned up, err: %v", test.secretName, err)
-				} else {
-					t.Fatalf("Failed to get secret %s, err: %v", test.secretName, err)
-				}
-			}
 		})
 	}
 }
 
+func waitConfigmapToBeLabeled(ctx context.Context, t *testing.T, c clientset.Interface) {
+	if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
+		configMap, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, legacytokentracking.ConfigMapName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		_, exist := configMap.Data[legacytokentracking.ConfigMapDataKey]
+		if !exist {
+			return false, fmt.Errorf("configMap does not have since label")
+		}
+		return true, nil
+	}); err != nil {
+		t.Fatalf("failed to wait configmap starts to track: %v", err)
+	}
+}
+
+func checkSecretCleanUp(ctx context.Context, t *testing.T, c clientset.Interface, secret *v1.Secret, shouldCleanUp bool) {
+	err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
+		_, err := c.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
+		if shouldCleanUp {
+			if err == nil {
+				return false, nil
+			} else if !apierrors.IsNotFound(err) {
+				t.Fatalf("Failed to get secret %s, err: %v", secret.Name, err)
+			}
+			return true, nil
+		}
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				t.Fatalf("The secret %s should not be cleaned up, err: %v", secret.Name, err)
+			} else {
+				t.Fatalf("Failed to get secret %s, err: %v", secret.Name, err)
+			}
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("Failed to check the existence for secret: %s, shouldCleanUp: %v, error: %v", secret.Name, shouldCleanUp, err)
+	}
+}
+
+func checkInvalidSinceLabel(ctx context.Context, t *testing.T, c clientset.Interface, secret *v1.Secret, fakeClock *testingclock.FakeClock, shouldLabel bool) {
+	err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
+		liveSecret, err := c.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
+		if err != nil {
+			t.Fatalf("Failed to get secret: %s, err: %v", secret.Name, err)
+		}
+		invalidSince, ok := liveSecret.GetLabels()[serviceaccount.InvalidSinceLabelKey]
+		if shouldLabel {
+			if !ok || invalidSince != fakeClock.Now().UTC().Format(dateFormat) {
+				return false, nil
+			}
+			return true, nil
+		}
+		if invalidSince != "" {
+			return false, nil
+		}
+		return true, nil
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to check secret invalid since label for secret: %s, shouldLabel: %v, error: %v", secret.Name, shouldLabel, err)
+	}
+}
+
+func checkLastUsedLabel(ctx context.Context, t *testing.T, c clientset.Interface, secret *v1.Secret, shouldLabel bool) {
+	err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
+		liveSecret, err := c.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
+		if err != nil {
+			t.Fatalf("Failed to get secret: %s, err: %v", secret.Name, err)
+		}
+		lastUsed, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey]
+		if shouldLabel {
+			if !ok || lastUsed != time.Now().UTC().Format(dateFormat) {
+				return false, nil
+			}
+			t.Logf("The secret %s has been labeled with %s", secret.Name, lastUsed)
+			return true, nil
+		}
+		if ok {
+			t.Fatalf("Secret %s should not have the lastUsed label", secret.Name)
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("Failed to check secret last used label for secret: %s, shouldLabel: %v, error: %v", secret.Name, shouldLabel, err)
+	}
+}
+
+func removeInvalidLabel(ctx context.Context, c clientset.Interface, t *testing.T, secret *v1.Secret) {
+	lastUsed := secret.GetLabels()[serviceaccount.LastUsedLabelKey]
+	patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithLabels(map[string]string{serviceaccount.InvalidSinceLabelKey: "", serviceaccount.LastUsedLabelKey: lastUsed}))
+	if err != nil {
+		t.Fatalf("Failed to marshal invalid since label, err: %v", err)
+	}
+	t.Logf("Patch the secret by removing the invalid label.")
+	if _, err := c.CoreV1().Secrets(secret.Namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
+		t.Fatalf("Failed to remove invalid since label, err: %v", err)
+	}
+	err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
+		secret, err = c.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
+		if err != nil {
+			t.Fatalf("Failed to get secret: %s, err: %v", secret.Name, err)
+		}
+		invalidSince := secret.GetLabels()[serviceaccount.InvalidSinceLabelKey]
+		if invalidSince != "" {
+			t.Log("Patch has not completed.")
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("Failed to patch secret: %s, err: %v", secret.Name, err)
+	}
+}
+
+func patchSecret(ctx context.Context, c clientset.Interface, t *testing.T, lastUsed string, secret *v1.Secret) {
+	patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithUID(secret.UID).WithLabels(map[string]string{serviceaccount.InvalidSinceLabelKey: "", serviceaccount.LastUsedLabelKey: lastUsed}))
+	if err != nil {
+		t.Fatalf("Failed to marshal invalid since label, err: %v", err)
+	}
+	t.Logf("Patch the secret by removing the invalid label.")
+	if _, err := c.CoreV1().Secrets(secret.Namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
+		t.Fatalf("Failed to remove invalid since label, err: %v", err)
+	}
+	err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
+		secret, err = c.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
+		if err != nil {
+			t.Fatalf("Failed to get secret: %s, err: %v", secret.Name, err)
+		}
+		lastUsedString := secret.GetLabels()[serviceaccount.LastUsedLabelKey]
+		if lastUsedString != lastUsed {
+			t.Log("Patch has not completed.")
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("Failed to patch secret: %s, err: %v", secret.Name, err)
+	}
+}
+
 func startLegacyServiceAccountTokenCleaner(ctx context.Context, client clientset.Interface, fakeClock clock.Clock, informers clientinformers.SharedInformerFactory) {
 	legacySATokenCleaner, _ := serviceaccountcontroller.NewLegacySATokenCleaner(
 		informers.Core().V1().ServiceAccounts(),
@@ -224,7 +359,33 @@ func startLegacyServiceAccountTokenCleaner(ctx context.Context, client clientset
 	go legacySATokenCleaner.Run(ctx)
 }
 
-func createAutotokenMountedPod(c clientset.Interface, ns, secretName string, podLister listersv1.PodLister) (*v1.Pod, error) {
+func doServiceAccountAPIReadRequest(ctx context.Context, t *testing.T, c clientset.Interface, ns string, authenticated bool) {
+	readOps := []testOperation{
+		func() error {
+			_, err := c.CoreV1().Secrets(ns).List(context.TODO(), metav1.ListOptions{})
+			return err
+		},
+		func() error {
+			_, err := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{})
+			return err
+		},
+	}
+
+	for _, op := range readOps {
+		err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
+			err := op()
+			if authenticated && err != nil || !authenticated && err == nil {
+				return false, nil
+			}
+			return true, nil
+		})
+		if err != nil {
+			t.Fatalf("Failed to check secret token authentication: error: %v", err)
+		}
+	}
+}
+
+func createAutotokenMountedPod(ctx context.Context, t *testing.T, c clientset.Interface, ns, secretName string, podLister listersv1.PodLister) *v1.Pod {
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "token-bound-pod",
@@ -239,14 +400,17 @@ func createAutotokenMountedPod(c clientset.Interface, ns, secretName string, pod
 	}
 	pod, err := c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("failed to create pod with token (%s:%s) bound, err: %v", ns, secretName, err)
+		t.Fatalf("Failed to create pod with token (%s:%s) bound, err: %v", ns, secretName, err)
 	}
-	err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
+	err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
 		pod, err = podLister.Pods(ns).Get("token-bound-pod")
 		if err != nil {
-			return false, fmt.Errorf("failed to get pod with token (%s:%s) bound, err: %v", ns, secretName, err)
+			return false, nil
 		}
 		return true, nil
 	})
-	return pod, nil
+	if err != nil {
+		t.Fatalf("Failed to wait auto-token mounted pod: err: %v", err)
+	}
+	return pod
 }