From 133eff3df4951a204a1acecd3a25afa3e3967c32 Mon Sep 17 00:00:00 2001 From: tinatingyu Date: Tue, 27 Dec 2022 17:23:05 +0000 Subject: [PATCH] implement LegacyServiceAccountTokenCleanUp alpha --- api/api-rules/violation_exceptions.list | 2 + .../app/controllermanager.go | 3 + cmd/kube-controller-manager/app/core.go | 20 ++ .../legacyserviceaccounttokencleaner.go | 58 +++ .../app/options/options.go | 9 + .../app/options/options_test.go | 18 + pkg/controller/apis/config/types.go | 2 + .../apis/config/v1alpha1/defaults.go | 2 + .../v1alpha1/zz_generated.conversion.go | 6 + .../apis/config/zz_generated.deepcopy.go | 1 + pkg/controller/serviceaccount/config/types.go | 10 + .../config/v1alpha1/conversion.go | 10 + .../config/v1alpha1/defaults.go | 10 + .../v1alpha1/zz_generated.conversion.go | 20 ++ .../config/zz_generated.deepcopy.go | 17 + .../legacy_serviceaccount_token_cleaner.go | 266 ++++++++++++++ ...egacy_serviceaccount_token_cleaner_test.go | 340 ++++++++++++++++++ pkg/features/kube_features.go | 9 + pkg/generated/openapi/zz_generated.openapi.go | 37 +- .../rbac/bootstrappolicy/controller_policy.go | 10 + .../config/v1alpha1/types.go | 11 +- .../config/v1alpha1/zz_generated.deepcopy.go | 18 + ...acy_service_account_token_clean_up_test.go | 251 +++++++++++++ .../serviceaccount/service_account_test.go | 20 +- 24 files changed, 1134 insertions(+), 16 deletions(-) create mode 100644 cmd/kube-controller-manager/app/options/legacyserviceaccounttokencleaner.go create mode 100644 pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner.go create mode 100644 pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner_test.go create mode 100644 test/integration/serviceaccount/legacy_service_account_token_clean_up_test.go diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index e745da8c7a5..21aeefa21a5 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -518,6 +518,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,K API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,HPAController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,JobController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,KubeCloudShared +API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,LegacySATokenCleaner API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,NamespaceController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,NodeIPAMController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,NodeLifecycleController @@ -530,6 +531,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,K API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,ServiceController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,StatefulSetController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,TTLAfterFinishedController +API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,LegacySATokenCleanerConfiguration,CleanUpPeriod API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,ConcurrentNamespaceSyncs API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,NamespaceSyncPeriod API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,NodeCIDRMaskSize diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 0b503c88b83..1d45611b15d 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -483,6 +483,9 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DynamicResourceAllocation) { register("resource-claim-controller", startResourceClaimController) } + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LegacyServiceAccountTokenCleanUp) { + register("legacy-service-account-token-cleaner", startLegacySATokenCleaner) + } return controllers } diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index eabdb24d82b..65e74a96f87 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -68,6 +68,7 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/pvprotection" quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" "k8s.io/kubernetes/pkg/volume/csimigration" + "k8s.io/utils/clock" netutils "k8s.io/utils/net" ) @@ -581,6 +582,25 @@ func startTTLAfterFinishedController(ctx context.Context, controllerContext Cont return nil, true, nil } +func startLegacySATokenCleaner(ctx context.Context, controllerContext ControllerContext) (controller.Interface, bool, error) { + cleanUpPeriod := controllerContext.ComponentConfig.LegacySATokenCleaner.CleanUpPeriod.Duration + legacySATokenCleaner, err := serviceaccountcontroller.NewLegacySATokenCleaner( + controllerContext.InformerFactory.Core().V1().ServiceAccounts(), + controllerContext.InformerFactory.Core().V1().Secrets(), + controllerContext.InformerFactory.Core().V1().Pods(), + controllerContext.ClientBuilder.ClientOrDie("legacy-service-account-token-cleaner"), + clock.RealClock{}, + serviceaccountcontroller.LegacySATokenCleanerOptions{ + CleanUpPeriod: cleanUpPeriod, + SyncInterval: serviceaccountcontroller.DefaultCleanerSyncInterval, + }) + if err != nil { + return nil, true, fmt.Errorf("failed to start the legacy service account token cleaner: %v", err) + } + go legacySATokenCleaner.Run(ctx) + return nil, true, nil +} + // processCIDRs is a helper function that works on a comma separated cidrs and returns // a list of typed cidrs // error if failed to parse any of the cidrs or invalid length of cidrs diff --git a/cmd/kube-controller-manager/app/options/legacyserviceaccounttokencleaner.go b/cmd/kube-controller-manager/app/options/legacyserviceaccounttokencleaner.go new file mode 100644 index 00000000000..ee38f797a96 --- /dev/null +++ b/cmd/kube-controller-manager/app/options/legacyserviceaccounttokencleaner.go @@ -0,0 +1,58 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "github.com/spf13/pflag" + + serviceaccountconfig "k8s.io/kubernetes/pkg/controller/serviceaccount/config" +) + +// LegacySATokenCleanerOptions holds the LegacySATokenCleaner options. +type LegacySATokenCleanerOptions struct { + *serviceaccountconfig.LegacySATokenCleanerConfiguration +} + +// AddFlags adds flags related to LegacySATokenCleaner for controller manager to the specified FlagSet +func (o *LegacySATokenCleanerOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.DurationVar(&o.CleanUpPeriod.Duration, "legacy-service-account-token-clean-up-period", o.CleanUpPeriod.Duration, "The period of time since the last usage of an legacy service account token before it can be deleted.") +} + +// ApplyTo fills up LegacySATokenCleaner config with options. +func (o *LegacySATokenCleanerOptions) ApplyTo(cfg *serviceaccountconfig.LegacySATokenCleanerConfiguration) error { + if o == nil { + return nil + } + + cfg.CleanUpPeriod = o.CleanUpPeriod + + return nil +} + +// Validate checks validation of LegacySATokenCleanerOptions. +func (o *LegacySATokenCleanerOptions) Validate() []error { + if o == nil { + return nil + } + + errs := []error{} + return errs +} diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 6c322f92521..1b92c0a5ff4 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -74,6 +74,7 @@ type KubeControllerManagerOptions struct { HPAController *HPAControllerOptions JobController *JobControllerOptions CronJobController *CronJobControllerOptions + LegacySATokenCleaner *LegacySATokenCleanerOptions NamespaceController *NamespaceControllerOptions NodeIPAMController *NodeIPAMControllerOptions NodeLifecycleController *NodeLifecycleControllerOptions @@ -150,6 +151,9 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { CronJobController: &CronJobControllerOptions{ &componentConfig.CronJobController, }, + LegacySATokenCleaner: &LegacySATokenCleanerOptions{ + &componentConfig.LegacySATokenCleaner, + }, NamespaceController: &NamespaceControllerOptions{ &componentConfig.NamespaceController, }, @@ -244,6 +248,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy s.HPAController.AddFlags(fss.FlagSet("horizontalpodautoscaling controller")) s.JobController.AddFlags(fss.FlagSet("job controller")) s.CronJobController.AddFlags(fss.FlagSet("cronjob controller")) + s.LegacySATokenCleaner.AddFlags(fss.FlagSet("legacy service account token cleaner")) s.NamespaceController.AddFlags(fss.FlagSet("namespace controller")) s.NodeIPAMController.AddFlags(fss.FlagSet("nodeipam controller")) s.NodeLifecycleController.AddFlags(fss.FlagSet("nodelifecycle controller")) @@ -315,6 +320,9 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e if err := s.CronJobController.ApplyTo(&c.ComponentConfig.CronJobController); err != nil { return err } + if err := s.LegacySATokenCleaner.ApplyTo(&c.ComponentConfig.LegacySATokenCleaner); err != nil { + return err + } if err := s.NamespaceController.ApplyTo(&c.ComponentConfig.NamespaceController); err != nil { return err } @@ -382,6 +390,7 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable errs = append(errs, s.HPAController.Validate()...) errs = append(errs, s.JobController.Validate()...) errs = append(errs, s.CronJobController.Validate()...) + errs = append(errs, s.LegacySATokenCleaner.Validate()...) errs = append(errs, s.NamespaceController.Validate()...) errs = append(errs, s.NodeIPAMController.Validate()...) errs = append(errs, s.NodeLifecycleController.Validate()...) diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index fac8128542a..c246412eb25 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -129,6 +129,7 @@ var args = []string{ "--leader-elect-renew-deadline=15s", "--leader-elect-resource-lock=configmap", "--leader-elect-retry-period=5s", + "--legacy-service-account-token-clean-up-period=8760h", "--master=192.168.4.20", "--max-endpoints-per-slice=200", "--min-resync-period=8h", @@ -397,6 +398,11 @@ func TestAddFlags(t *testing.T) { ConcurrentSATokenSyncs: 10, }, }, + LegacySATokenCleaner: &LegacySATokenCleanerOptions{ + &serviceaccountconfig.LegacySATokenCleanerConfiguration{ + CleanUpPeriod: metav1.Duration{Duration: 365 * 24 * time.Hour}, + }, + }, TTLAfterFinishedController: &TTLAfterFinishedControllerOptions{ &ttlafterfinishedconfig.TTLAfterFinishedControllerConfiguration{ ConcurrentTTLSyncs: 8, @@ -627,6 +633,9 @@ func TestApplyTo(t *testing.T) { ServiceAccountKeyFile: "/service-account-private-key", ConcurrentSATokenSyncs: 10, }, + LegacySATokenCleaner: serviceaccountconfig.LegacySATokenCleanerConfiguration{ + CleanUpPeriod: metav1.Duration{Duration: 365 * 24 * time.Hour}, + }, TTLAfterFinishedController: ttlafterfinishedconfig.TTLAfterFinishedControllerConfiguration{ ConcurrentTTLSyncs: 8, }, @@ -1225,6 +1234,15 @@ func TestValidateControllersOptions(t *testing.T) { }, }).Validate, }, + { + name: "LegacySATokenCleanerOptions", + expectErrors: false, + validate: (&LegacySATokenCleanerOptions{ + &serviceaccountconfig.LegacySATokenCleanerConfiguration{ + CleanUpPeriod: metav1.Duration{Duration: 24 * 365 * time.Hour}, + }, + }).Validate, + }, { name: "TTLAfterFinishedControllerOptions", expectErrors: false, diff --git a/pkg/controller/apis/config/types.go b/pkg/controller/apis/config/types.go index 9ec93e3c33b..108bf81a7e3 100644 --- a/pkg/controller/apis/config/types.go +++ b/pkg/controller/apis/config/types.go @@ -98,6 +98,8 @@ type KubeControllerManagerConfiguration struct { // CronJobControllerConfiguration holds configuration for CronJobController // related features. CronJobController cronjobconfig.CronJobControllerConfiguration + // LegacySATokenCleanerConfiguration holds configuration for LegacySATokenCleaner related features. + LegacySATokenCleaner serviceaccountconfig.LegacySATokenCleanerConfiguration // NamespaceControllerConfiguration holds configuration for NamespaceController // related features. NamespaceController namespaceconfig.NamespaceControllerConfiguration diff --git a/pkg/controller/apis/config/v1alpha1/defaults.go b/pkg/controller/apis/config/v1alpha1/defaults.go index 14b177073d0..078f8edc63b 100644 --- a/pkg/controller/apis/config/v1alpha1/defaults.go +++ b/pkg/controller/apis/config/v1alpha1/defaults.go @@ -104,6 +104,8 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *kubectrlmgrconfigv1alph resourcequotaconfigv1alpha1.RecommendedDefaultResourceQuotaControllerConfiguration(&obj.ResourceQuotaController) // Use the default RecommendedDefaultGenericControllerManagerConfiguration options serviceconfigv1alpha1.RecommendedDefaultServiceControllerConfiguration(&obj.ServiceController) + // Use the default RecommendedDefaultLegacySATokenCleanerConfiguration options + serviceaccountconfigv1alpha1.RecommendedDefaultLegacySATokenCleanerConfiguration(&obj.LegacySATokenCleaner) // Use the default RecommendedDefaultSAControllerConfiguration options serviceaccountconfigv1alpha1.RecommendedDefaultSAControllerConfiguration(&obj.SAController) // Use the default RecommendedDefaultTTLAfterFinishedControllerConfiguration options diff --git a/pkg/controller/apis/config/v1alpha1/zz_generated.conversion.go b/pkg/controller/apis/config/v1alpha1/zz_generated.conversion.go index e3f30346bce..91f54cc6171 100644 --- a/pkg/controller/apis/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/controller/apis/config/v1alpha1/zz_generated.conversion.go @@ -184,6 +184,9 @@ func autoConvert_v1alpha1_KubeControllerManagerConfiguration_To_config_KubeContr if err := cronjobconfigv1alpha1.Convert_v1alpha1_CronJobControllerConfiguration_To_config_CronJobControllerConfiguration(&in.CronJobController, &out.CronJobController, s); err != nil { return err } + if err := serviceaccountconfigv1alpha1.Convert_v1alpha1_LegacySATokenCleanerConfiguration_To_config_LegacySATokenCleanerConfiguration(&in.LegacySATokenCleaner, &out.LegacySATokenCleaner, s); err != nil { + return err + } if err := namespaceconfigv1alpha1.Convert_v1alpha1_NamespaceControllerConfiguration_To_config_NamespaceControllerConfiguration(&in.NamespaceController, &out.NamespaceController, s); err != nil { return err } @@ -274,6 +277,9 @@ func autoConvert_config_KubeControllerManagerConfiguration_To_v1alpha1_KubeContr if err := cronjobconfigv1alpha1.Convert_config_CronJobControllerConfiguration_To_v1alpha1_CronJobControllerConfiguration(&in.CronJobController, &out.CronJobController, s); err != nil { return err } + if err := serviceaccountconfigv1alpha1.Convert_config_LegacySATokenCleanerConfiguration_To_v1alpha1_LegacySATokenCleanerConfiguration(&in.LegacySATokenCleaner, &out.LegacySATokenCleaner, s); err != nil { + return err + } if err := namespaceconfigv1alpha1.Convert_config_NamespaceControllerConfiguration_To_v1alpha1_NamespaceControllerConfiguration(&in.NamespaceController, &out.NamespaceController, s); err != nil { return err } diff --git a/pkg/controller/apis/config/zz_generated.deepcopy.go b/pkg/controller/apis/config/zz_generated.deepcopy.go index 61cc1cbbdc9..10aca159bae 100644 --- a/pkg/controller/apis/config/zz_generated.deepcopy.go +++ b/pkg/controller/apis/config/zz_generated.deepcopy.go @@ -61,6 +61,7 @@ func (in *KubeControllerManagerConfiguration) DeepCopyInto(out *KubeControllerMa out.HPAController = in.HPAController out.JobController = in.JobController out.CronJobController = in.CronJobController + out.LegacySATokenCleaner = in.LegacySATokenCleaner out.NamespaceController = in.NamespaceController out.NodeIPAMController = in.NodeIPAMController out.NodeLifecycleController = in.NodeLifecycleController diff --git a/pkg/controller/serviceaccount/config/types.go b/pkg/controller/serviceaccount/config/types.go index 2bf464d8f8b..49228178548 100644 --- a/pkg/controller/serviceaccount/config/types.go +++ b/pkg/controller/serviceaccount/config/types.go @@ -16,6 +16,10 @@ limitations under the License. package config +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + // SAControllerConfiguration contains elements describing ServiceAccountController. type SAControllerConfiguration struct { // serviceAccountKeyFile is the filename containing a PEM-encoded private RSA key @@ -28,3 +32,9 @@ type SAControllerConfiguration struct { // account's token secret. This must be a valid PEM-encoded CA bundle. RootCAFile string } + +type LegacySATokenCleanerConfiguration struct { + // CleanUpPeriod is the period of time since the last usage of an + // auto-generated service account token before it can be deleted. + CleanUpPeriod metav1.Duration +} diff --git a/pkg/controller/serviceaccount/config/v1alpha1/conversion.go b/pkg/controller/serviceaccount/config/v1alpha1/conversion.go index 1187195cb3f..6204a1810e6 100644 --- a/pkg/controller/serviceaccount/config/v1alpha1/conversion.go +++ b/pkg/controller/serviceaccount/config/v1alpha1/conversion.go @@ -38,3 +38,13 @@ func Convert_v1alpha1_SAControllerConfiguration_To_config_SAControllerConfigurat func Convert_config_SAControllerConfiguration_To_v1alpha1_SAControllerConfiguration(in *config.SAControllerConfiguration, out *v1alpha1.SAControllerConfiguration, s conversion.Scope) error { return autoConvert_config_SAControllerConfiguration_To_v1alpha1_SAControllerConfiguration(in, out, s) } + +// Convert_v1alpha1_LegacySATokenCleanerConfiguration_To_config_LegacySATokenCleanerConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_LegacySATokenCleanerConfiguration_To_config_LegacySATokenCleanerConfiguration(in *v1alpha1.LegacySATokenCleanerConfiguration, out *config.LegacySATokenCleanerConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_LegacySATokenCleanerConfiguration_To_config_LegacySATokenCleanerConfiguration(in, out, s) +} + +// Convert_config_LegacySATokenCleanerConfiguration_To_v1alpha1_LegacySATokenCleanerConfiguration is an autogenerated conversion function. +func Convert_config_LegacySATokenCleanerConfiguration_To_v1alpha1_LegacySATokenCleanerConfiguration(in *config.LegacySATokenCleanerConfiguration, out *v1alpha1.LegacySATokenCleanerConfiguration, s conversion.Scope) error { + return autoConvert_config_LegacySATokenCleanerConfiguration_To_v1alpha1_LegacySATokenCleanerConfiguration(in, out, s) +} diff --git a/pkg/controller/serviceaccount/config/v1alpha1/defaults.go b/pkg/controller/serviceaccount/config/v1alpha1/defaults.go index 59c628225b1..17c9afa2f7c 100644 --- a/pkg/controller/serviceaccount/config/v1alpha1/defaults.go +++ b/pkg/controller/serviceaccount/config/v1alpha1/defaults.go @@ -17,6 +17,9 @@ limitations under the License. package v1alpha1 import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubectrlmgrconfigv1alpha1 "k8s.io/kube-controller-manager/config/v1alpha1" ) @@ -34,3 +37,10 @@ func RecommendedDefaultSAControllerConfiguration(obj *kubectrlmgrconfigv1alpha1. obj.ConcurrentSATokenSyncs = 5 } } + +func RecommendedDefaultLegacySATokenCleanerConfiguration(obj *kubectrlmgrconfigv1alpha1.LegacySATokenCleanerConfiguration) { + zero := metav1.Duration{} + if obj.CleanUpPeriod == zero { + obj.CleanUpPeriod = metav1.Duration{Duration: 365 * 24 * time.Hour} + } +} diff --git a/pkg/controller/serviceaccount/config/v1alpha1/zz_generated.conversion.go b/pkg/controller/serviceaccount/config/v1alpha1/zz_generated.conversion.go index 72b3fca95f1..d416609aefb 100644 --- a/pkg/controller/serviceaccount/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/controller/serviceaccount/config/v1alpha1/zz_generated.conversion.go @@ -46,11 +46,21 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*config.LegacySATokenCleanerConfiguration)(nil), (*v1alpha1.LegacySATokenCleanerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_LegacySATokenCleanerConfiguration_To_v1alpha1_LegacySATokenCleanerConfiguration(a.(*config.LegacySATokenCleanerConfiguration), b.(*v1alpha1.LegacySATokenCleanerConfiguration), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*config.SAControllerConfiguration)(nil), (*v1alpha1.SAControllerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_config_SAControllerConfiguration_To_v1alpha1_SAControllerConfiguration(a.(*config.SAControllerConfiguration), b.(*v1alpha1.SAControllerConfiguration), scope) }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha1.LegacySATokenCleanerConfiguration)(nil), (*config.LegacySATokenCleanerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_LegacySATokenCleanerConfiguration_To_config_LegacySATokenCleanerConfiguration(a.(*v1alpha1.LegacySATokenCleanerConfiguration), b.(*config.LegacySATokenCleanerConfiguration), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha1.SAControllerConfiguration)(nil), (*config.SAControllerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_SAControllerConfiguration_To_config_SAControllerConfiguration(a.(*v1alpha1.SAControllerConfiguration), b.(*config.SAControllerConfiguration), scope) }); err != nil { @@ -81,6 +91,16 @@ func Convert_v1_GroupResource_To_v1alpha1_GroupResource(in *v1.GroupResource, ou return autoConvert_v1_GroupResource_To_v1alpha1_GroupResource(in, out, s) } +func autoConvert_v1alpha1_LegacySATokenCleanerConfiguration_To_config_LegacySATokenCleanerConfiguration(in *v1alpha1.LegacySATokenCleanerConfiguration, out *config.LegacySATokenCleanerConfiguration, s conversion.Scope) error { + out.CleanUpPeriod = in.CleanUpPeriod + return nil +} + +func autoConvert_config_LegacySATokenCleanerConfiguration_To_v1alpha1_LegacySATokenCleanerConfiguration(in *config.LegacySATokenCleanerConfiguration, out *v1alpha1.LegacySATokenCleanerConfiguration, s conversion.Scope) error { + out.CleanUpPeriod = in.CleanUpPeriod + return nil +} + func autoConvert_v1alpha1_SAControllerConfiguration_To_config_SAControllerConfiguration(in *v1alpha1.SAControllerConfiguration, out *config.SAControllerConfiguration, s conversion.Scope) error { out.ServiceAccountKeyFile = in.ServiceAccountKeyFile out.ConcurrentSATokenSyncs = in.ConcurrentSATokenSyncs diff --git a/pkg/controller/serviceaccount/config/zz_generated.deepcopy.go b/pkg/controller/serviceaccount/config/zz_generated.deepcopy.go index 72908efe91e..4e1662b9feb 100644 --- a/pkg/controller/serviceaccount/config/zz_generated.deepcopy.go +++ b/pkg/controller/serviceaccount/config/zz_generated.deepcopy.go @@ -21,6 +21,23 @@ limitations under the License. package config +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LegacySATokenCleanerConfiguration) DeepCopyInto(out *LegacySATokenCleanerConfiguration) { + *out = *in + out.CleanUpPeriod = in.CleanUpPeriod + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LegacySATokenCleanerConfiguration. +func (in *LegacySATokenCleanerConfiguration) DeepCopy() *LegacySATokenCleanerConfiguration { + if in == nil { + return nil + } + out := new(LegacySATokenCleanerConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SAControllerConfiguration) DeepCopyInto(out *SAControllerConfiguration) { *out = *in diff --git a/pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner.go b/pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner.go new file mode 100644 index 00000000000..faafb33a67e --- /dev/null +++ b/pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner.go @@ -0,0 +1,266 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serviceaccount + +import ( + "context" + "fmt" + "time" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + listersv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking" + "k8s.io/kubernetes/pkg/serviceaccount" + "k8s.io/utils/clock" +) + +const ( + dateFormat = "2006-01-02" + DefaultCleanerSyncInterval = 24 * time.Hour +) + +// TokenCleanerOptions contains options for the LegacySATokenCleaner +type LegacySATokenCleanerOptions struct { + // CleanUpPeriod is the period of time since the last usage of an legacy token before it can be deleted. + CleanUpPeriod time.Duration + SyncInterval time.Duration +} + +// LegacySATokenCleaner is a controller that deletes legacy serviceaccount tokens that are not in use for a specified period of time. +type LegacySATokenCleaner struct { + client clientset.Interface + clock clock.Clock + saLister listersv1.ServiceAccountLister + saInformerSynced cache.InformerSynced + + secretLister listersv1.SecretLister + secretInformerSynced cache.InformerSynced + + podLister listersv1.PodLister + podInformerSynced cache.InformerSynced + + syncInterval time.Duration + minimumSinceLastUsed time.Duration +} + +// NewLegacySATokenCleaner returns a new *NewLegacySATokenCleaner. +func NewLegacySATokenCleaner(saInformer coreinformers.ServiceAccountInformer, secretInformer coreinformers.SecretInformer, podInformer coreinformers.PodInformer, client clientset.Interface, cl clock.Clock, options LegacySATokenCleanerOptions) (*LegacySATokenCleaner, error) { + if !(options.CleanUpPeriod > 0) { + return nil, fmt.Errorf("invalid CleanUpPeriod: %v", options.CleanUpPeriod) + } + if !(options.SyncInterval > 0) { + return nil, fmt.Errorf("invalid SyncInterval: %v", options.SyncInterval) + } + + tc := &LegacySATokenCleaner{ + client: client, + clock: cl, + saLister: saInformer.Lister(), + saInformerSynced: saInformer.Informer().HasSynced, + secretLister: secretInformer.Lister(), + secretInformerSynced: secretInformer.Informer().HasSynced, + podLister: podInformer.Lister(), + podInformerSynced: podInformer.Informer().HasSynced, + minimumSinceLastUsed: options.CleanUpPeriod, + syncInterval: options.SyncInterval, + } + + return tc, nil +} + +func (tc *LegacySATokenCleaner) Run(ctx context.Context) { + defer utilruntime.HandleCrash() + + logger := klog.FromContext(ctx) + logger.Info("Starting legacy service account token cleaner controller") + defer logger.Info("Shutting down legacy service account token cleaner controller") + + if !cache.WaitForNamedCacheSync("legacy-service-account-token-cleaner", ctx.Done(), tc.saInformerSynced, tc.secretInformerSynced, tc.podInformerSynced) { + return + } + + go wait.UntilWithContext(ctx, tc.evaluateSATokens, tc.syncInterval) + + <-ctx.Done() +} + +func (tc *LegacySATokenCleaner) evaluateSATokens(ctx context.Context) { + logger := klog.FromContext(ctx) + + now := tc.clock.Now().UTC() + trackedSince, err := tc.latestPossibleTrackedSinceTime(ctx) + if err != nil { + logger.Error(err, "Getting lastest possible tracked_since time") + return + } + + if now.Before(trackedSince.Add(tc.minimumSinceLastUsed)) { + // we haven't been tracking long enough + return + } + + preserveCreatedOnOrAfter := now.Add(-tc.minimumSinceLastUsed) + preserveUsedOnOrAfter := now.Add(-tc.minimumSinceLastUsed).Format(dateFormat) + + secretList, err := tc.secretLister.Secrets(metav1.NamespaceAll).List(labels.Everything()) + if err != nil { + logger.Error(err, "Getting cached secret list") + return + } + + namespaceToUsedSecretNames := make(map[string]sets.String) + for _, secret := range secretList { + if secret.Type != v1.SecretTypeServiceAccountToken { + continue + } + if !secret.CreationTimestamp.Time.Before(preserveCreatedOnOrAfter) { + continue + } + + if secret.DeletionTimestamp != nil { + continue + } + + // if LastUsedLabelKey does not exist, we think the secret has not been used + // since the legacy token starts to track. + lastUsed, ok := secret.Labels[serviceaccount.LastUsedLabelKey] + if ok { + _, err := time.Parse(dateFormat, lastUsed) + if err != nil { + // the lastUsed value is not well-formed thus we cannot determine it + logger.Error(err, "Parsing lastUsed time", "secret", klog.KRef(secret.Namespace, secret.Name)) + continue + } + if lastUsed >= preserveUsedOnOrAfter { + continue + } + } + + sa, saErr := tc.getServiceAccount(secret) + + if saErr != nil { + logger.Error(saErr, "Getting service account", "secret", klog.KRef(secret.Namespace, secret.Name)) + continue + } + if sa == nil || !hasSecretReference(sa, secret.Name) { + // can't determine if this is an auto-generated token + continue + } + + mountedSecretNames, err := tc.getMountedSecretNames(secret.Namespace, namespaceToUsedSecretNames) + if err != nil { + logger.Error(err, "Resolving mounted secrets", "secret", klog.KRef(secret.Namespace, secret.Name)) + continue + } + if mountedSecretNames.Has(secret.Name) { + // still used by pods + continue + } + + logger.Info("Delete auto-generated service account token", "secret", klog.KRef(secret.Namespace, secret.Name), "creationTime", secret.CreationTimestamp, "lastUsed", lastUsed) + if err := tc.client.CoreV1().Secrets(secret.Namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &secret.ResourceVersion}}); err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { + logger.Error(err, "Deleting legacy service account token", "secret", klog.KRef(secret.Namespace, secret.Name), "serviceaccount", sa.Name) + } + } +} + +func (tc *LegacySATokenCleaner) getMountedSecretNames(secretNamespace string, namespaceToUsedSecretNames map[string]sets.String) (sets.String, error) { + if secrets, ok := namespaceToUsedSecretNames[secretNamespace]; ok { + return secrets, nil + } + + podList, err := tc.podLister.Pods(secretNamespace).List(labels.Everything()) + if err != nil { + return nil, fmt.Errorf("failed to get pod list from pod cache: %v", err) + } + + var secrets sets.String + for _, pod := range podList { + podutil.VisitPodSecretNames(pod, func(secretName string) bool { + if secrets == nil { + secrets = sets.NewString() + } + secrets.Insert(secretName) + return true + }) + } + if secrets != nil { + namespaceToUsedSecretNames[secretNamespace] = secrets + } + return secrets, nil +} + +func (tc *LegacySATokenCleaner) getServiceAccount(secret *v1.Secret) (*v1.ServiceAccount, error) { + saName := secret.Annotations[v1.ServiceAccountNameKey] + if len(saName) == 0 { + return nil, nil + } + saUID := types.UID(secret.Annotations[v1.ServiceAccountUIDKey]) + sa, err := tc.saLister.ServiceAccounts(secret.Namespace).Get(saName) + if apierrors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + + // Ensure UID matches if given + if len(saUID) == 0 || saUID == sa.UID { + return sa, nil + } + + return nil, nil +} + +// get the latest possible TrackedSince time information from the configMap label. +func (tc *LegacySATokenCleaner) latestPossibleTrackedSinceTime(ctx context.Context) (time.Time, error) { + configMap, err := tc.client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, legacytokentracking.ConfigMapName, metav1.GetOptions{}) + if err != nil { + return time.Time{}, err + } + trackedSince, exist := configMap.Data[legacytokentracking.ConfigMapDataKey] + if !exist { + return time.Time{}, fmt.Errorf("configMap does not have since label") + } + trackedSinceTime, err := time.Parse(dateFormat, trackedSince) + if err != nil { + return time.Time{}, fmt.Errorf("error parsing trackedSince time: %v", err) + } + // make sure the time to be 00:00 on the day just after the date starts to track + return trackedSinceTime.AddDate(0, 0, 1), nil +} + +func hasSecretReference(serviceAccount *v1.ServiceAccount, secretName string) bool { + for _, secret := range serviceAccount.Secrets { + if secret.Name == secretName { + return true + } + } + return false +} diff --git a/pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner_test.go b/pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner_test.go new file mode 100644 index 00000000000..82a358e5a91 --- /dev/null +++ b/pkg/controller/serviceaccount/legacy_serviceaccount_token_cleaner_test.go @@ -0,0 +1,340 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serviceaccount + +import ( + "context" + "reflect" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" + "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking" + "k8s.io/kubernetes/pkg/serviceaccount" + testingclock "k8s.io/utils/clock/testing" +) + +func configuredConfigMap(label string) *v1.ConfigMap { + if label == "" { + return &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: legacytokentracking.ConfigMapName}, + } + } + return &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: legacytokentracking.ConfigMapName}, + Data: map[string]string{legacytokentracking.ConfigMapDataKey: label}, + } +} + +func configuredServiceAccountTokenSecret(label, creationTimeString, serviceAccountName, serviceAccountUID, deletionTimeString string) *v1.Secret { + var deletionTime *metav1.Time + if deletionTimeString == "" { + deletionTime = nil + } else { + deletionTime = &metav1.Time{Time: time.Now().UTC()} + } + creationTime, _ := time.Parse(dateFormat, creationTimeString) + labels := map[string]string{} + if label != "" { + labels[serviceaccount.LastUsedLabelKey] = label + } + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "token-secret-1", + Namespace: "default", + UID: "23456", + ResourceVersion: "1", + Labels: labels, + CreationTimestamp: metav1.NewTime(creationTime), + DeletionTimestamp: deletionTime, + Annotations: map[string]string{ + v1.ServiceAccountNameKey: serviceAccountName, + v1.ServiceAccountUIDKey: serviceAccountUID, + }, + }, + Type: v1.SecretTypeServiceAccountToken, + Data: map[string][]byte{ + "token": []byte("ABC"), + "ca.crt": []byte("CA Data"), + "namespace": []byte("default"), + }, + } +} + +func configuredLegacyTokenCleanUpPeriod(start string) time.Duration { + current := time.Now().UTC() + startTime, _ := time.Parse(dateFormat, start) + return current.Sub(startTime) +} + +func configuredPod(withSecretMount bool) *v1.Pod { + if !withSecretMount { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "default", + }, + } + } + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{{Name: "foo", VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: "token-secret-1"}}}}, + }, + } +} + +func TestLegacyServiceAccountTokenCleanUp(t *testing.T) { + testcases := map[string]struct { + LegacyTokenCleanUpPeriod time.Duration + + ExistingServiceAccount *v1.ServiceAccount + ExistingSecret *v1.Secret + ExistingPod *v1.Pod + ClientObjects []runtime.Object + + ExpectedActions []core.Action + }{ + "configmap does not exist": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-28"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "configmap exists, but the configmap does not have tracked-since label": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-28"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "configmap exists, the time period since 'tracked-since' is smaller than the CleanUpPeriod": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-29")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "configmap exists, the 'tracked-since' cannot be parsed": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27-1")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "secret is not SecretTypeServiceAccountToken type": { + ExistingSecret: opaqueSecret(), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-28")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "secret is not referenced by serviceaccount": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(emptySecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "auto-generated secret has a late creation time": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-30", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "auto-generated secret has a deletion time": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", "deleted"), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-30"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "auto-generated secret has a late last-used time": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-30", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "auto-generated secret has a last-used label, but it can not be parsed": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27-1", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "secret-referenced service account does not exist": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "secret-referenced service account uid does not match": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "123456", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "secret-referenced service account name is empty": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-27")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "auto-generated secret does not have 'last-used' label": { + ExistingSecret: configuredServiceAccountTokenSecret("", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-28")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-30"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + core.NewDeleteActionWithOptions( + schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, + metav1.NamespaceDefault, "token-secret-1", + metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{ResourceVersion: &configuredServiceAccountTokenSecret("", "2022-12-27", "default", "12345", "").ResourceVersion}, + }), + }, + }, + "auto-generated secret is mounted by the pod": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(true), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-28")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-29"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + }, + }, + "auto-generated secret has 'last-used' label, the time period since last-used is larger than CleanUpPeriod": { + ExistingSecret: configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", ""), + ExistingServiceAccount: serviceAccount(tokenSecretReferences()), + ExistingPod: configuredPod(false), + ClientObjects: []runtime.Object{configuredConfigMap("2022-12-28")}, + LegacyTokenCleanUpPeriod: configuredLegacyTokenCleanUpPeriod("2022-12-30"), + ExpectedActions: []core.Action{ + core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, metav1.NamespaceSystem, legacytokentracking.ConfigMapName), + core.NewDeleteActionWithOptions( + schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, + metav1.NamespaceDefault, "token-secret-1", + metav1.DeleteOptions{Preconditions: &metav1.Preconditions{ + ResourceVersion: &configuredServiceAccountTokenSecret("2022-12-27", "2022-12-27", "default", "12345", "").ResourceVersion, + }, + }), + }, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + tc.ClientObjects = append(tc.ClientObjects, tc.ExistingSecret) + client := fake.NewSimpleClientset(tc.ClientObjects...) + + informers := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) + secretInformer := informers.Core().V1().Secrets() + saInformer := informers.Core().V1().ServiceAccounts() + podInformer := informers.Core().V1().Pods() + secrets := secretInformer.Informer().GetStore() + serviceAccounts := saInformer.Informer().GetStore() + pods := podInformer.Informer().GetStore() + options := LegacySATokenCleanerOptions{ + SyncInterval: 30 * time.Second, + CleanUpPeriod: tc.LegacyTokenCleanUpPeriod, + } + cleaner, _ := NewLegacySATokenCleaner(saInformer, secretInformer, podInformer, client, testingclock.NewFakeClock(time.Now().UTC()), options) + + if tc.ExistingServiceAccount != nil { + serviceAccounts.Add(tc.ExistingServiceAccount) + } + if tc.ExistingPod != nil { + pods.Add(tc.ExistingPod) + } + secrets.Add(tc.ExistingSecret) + + ctx := context.TODO() + cleaner.evaluateSATokens(ctx) + + actions := client.Actions() + if len(actions) != len(tc.ExpectedActions) { + t.Fatalf("got %d actions, wanted %d actions", len(actions), len(tc.ExpectedActions)) + } + for i, action := range actions { + if len(tc.ExpectedActions) < i+1 { + t.Errorf("%s: %d unexpected actions: %+v", k, len(actions)-len(tc.ExpectedActions), actions[i:]) + break + } + expectedAction := tc.ExpectedActions[i] + if !reflect.DeepEqual(expectedAction, action) { + t.Errorf("got action %#v, wanted %v", action, expectedAction) + } + } + }) + } +} diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 2fb76d2bf4f..758150e8721 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -466,6 +466,13 @@ const ( // Enables tracking of secret-based service account tokens usage. LegacyServiceAccountTokenTracking featuregate.Feature = "LegacyServiceAccountTokenTracking" + // owner: @yt2985 + // kep: http://kep.k8s.io/2800 + // alpha: v1.28 + // + // Enables cleaning up of secret-based service account tokens. + LegacyServiceAccountTokenCleanUp featuregate.Feature = "LegacyServiceAccountTokenCleanUp" + // owner: @RobertKrawitz // alpha: v1.15 // @@ -962,6 +969,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS LegacyServiceAccountTokenTracking: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.30 + LegacyServiceAccountTokenCleanUp: {Default: false, PreRelease: featuregate.Alpha}, + LocalStorageCapacityIsolationFSQuotaMonitoring: {Default: false, PreRelease: featuregate.Alpha}, LogarithmicScaleDown: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index b7dad245930..48ce6294538 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1064,6 +1064,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/kube-controller-manager/config/v1alpha1.HPAControllerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_HPAControllerConfiguration(ref), "k8s.io/kube-controller-manager/config/v1alpha1.JobControllerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_JobControllerConfiguration(ref), "k8s.io/kube-controller-manager/config/v1alpha1.KubeControllerManagerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_KubeControllerManagerConfiguration(ref), + "k8s.io/kube-controller-manager/config/v1alpha1.LegacySATokenCleanerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_LegacySATokenCleanerConfiguration(ref), "k8s.io/kube-controller-manager/config/v1alpha1.NamespaceControllerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_NamespaceControllerConfiguration(ref), "k8s.io/kube-controller-manager/config/v1alpha1.NodeIPAMControllerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_NodeIPAMControllerConfiguration(ref), "k8s.io/kube-controller-manager/config/v1alpha1.NodeLifecycleControllerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_NodeLifecycleControllerConfiguration(ref), @@ -53044,9 +53045,16 @@ func schema_k8sio_kube_controller_manager_config_v1alpha1_KubeControllerManagerC Ref: ref("k8s.io/kube-controller-manager/config/v1alpha1.CronJobControllerConfiguration"), }, }, + "LegacySATokenCleaner": { + SchemaProps: spec.SchemaProps{ + Description: "LegacySATokenCleanerConfiguration holds configuration for LegacySATokenCleaner related features.", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/kube-controller-manager/config/v1alpha1.LegacySATokenCleanerConfiguration"), + }, + }, "NamespaceController": { SchemaProps: spec.SchemaProps{ - Description: "NamespaceControllerConfiguration holds configuration for NamespaceController related features. NamespaceControllerConfiguration holds configuration for NamespaceController related features.", + Description: "NamespaceControllerConfiguration holds configuration for NamespaceController related features.", Default: map[string]interface{}{}, Ref: ref("k8s.io/kube-controller-manager/config/v1alpha1.NamespaceControllerConfiguration"), }, @@ -53122,11 +53130,34 @@ func schema_k8sio_kube_controller_manager_config_v1alpha1_KubeControllerManagerC }, }, }, - Required: []string{"Generic", "KubeCloudShared", "AttachDetachController", "CSRSigningController", "DaemonSetController", "DeploymentController", "StatefulSetController", "DeprecatedController", "EndpointController", "EndpointSliceController", "EndpointSliceMirroringController", "EphemeralVolumeController", "GarbageCollectorController", "HPAController", "JobController", "CronJobController", "NamespaceController", "NodeIPAMController", "NodeLifecycleController", "PersistentVolumeBinderController", "PodGCController", "ReplicaSetController", "ReplicationController", "ResourceQuotaController", "SAController", "ServiceController", "TTLAfterFinishedController"}, + Required: []string{"Generic", "KubeCloudShared", "AttachDetachController", "CSRSigningController", "DaemonSetController", "DeploymentController", "StatefulSetController", "DeprecatedController", "EndpointController", "EndpointSliceController", "EndpointSliceMirroringController", "EphemeralVolumeController", "GarbageCollectorController", "HPAController", "JobController", "CronJobController", "LegacySATokenCleaner", "NamespaceController", "NodeIPAMController", "NodeLifecycleController", "PersistentVolumeBinderController", "PodGCController", "ReplicaSetController", "ReplicationController", "ResourceQuotaController", "SAController", "ServiceController", "TTLAfterFinishedController"}, }, }, Dependencies: []string{ - "k8s.io/cloud-provider/config/v1alpha1.KubeCloudSharedConfiguration", "k8s.io/cloud-provider/controllers/service/config/v1alpha1.ServiceControllerConfiguration", "k8s.io/controller-manager/config/v1alpha1.GenericControllerManagerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.AttachDetachControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.CSRSigningControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.CronJobControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.DaemonSetControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.DeploymentControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.DeprecatedControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.EndpointControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.EndpointSliceControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.EndpointSliceMirroringControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.EphemeralVolumeControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.GarbageCollectorControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.HPAControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.JobControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.NamespaceControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.NodeIPAMControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.NodeLifecycleControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.PersistentVolumeBinderControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.PodGCControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.ReplicaSetControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.ReplicationControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.ResourceQuotaControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.SAControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.StatefulSetControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.TTLAfterFinishedControllerConfiguration"}, + "k8s.io/cloud-provider/config/v1alpha1.KubeCloudSharedConfiguration", "k8s.io/cloud-provider/controllers/service/config/v1alpha1.ServiceControllerConfiguration", "k8s.io/controller-manager/config/v1alpha1.GenericControllerManagerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.AttachDetachControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.CSRSigningControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.CronJobControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.DaemonSetControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.DeploymentControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.DeprecatedControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.EndpointControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.EndpointSliceControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.EndpointSliceMirroringControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.EphemeralVolumeControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.GarbageCollectorControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.HPAControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.JobControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.LegacySATokenCleanerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.NamespaceControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.NodeIPAMControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.NodeLifecycleControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.PersistentVolumeBinderControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.PodGCControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.ReplicaSetControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.ReplicationControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.ResourceQuotaControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.SAControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.StatefulSetControllerConfiguration", "k8s.io/kube-controller-manager/config/v1alpha1.TTLAfterFinishedControllerConfiguration"}, + } +} + +func schema_k8sio_kube_controller_manager_config_v1alpha1_LegacySATokenCleanerConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "LegacySATokenCleanerConfiguration contains elements describing LegacySATokenCleaner", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "CleanUpPeriod": { + SchemaProps: spec.SchemaProps{ + Description: "CleanUpPeriod is the period of time since the last usage of an auto-generated service account token before it can be deleted.", + Default: 0, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + }, + Required: []string{"CleanUpPeriod"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.Duration"}, } } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index 8f9d5016294..5e1b3f85486 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -27,6 +27,7 @@ import ( genericfeatures "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1" + "k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking" "k8s.io/kubernetes/pkg/features" ) @@ -452,6 +453,15 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding) }, }) } + if utilfeature.DefaultFeatureGate.Enabled(features.LegacyServiceAccountTokenCleanUp) { + addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "legacy-service-account-token-cleaner"}, + Rules: []rbacv1.PolicyRule{ + rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("configmaps").Names(legacytokentracking.ConfigMapName).RuleOrDie(), + rbacv1helpers.NewRule("delete").Groups(legacyGroup).Resources("secrets").RuleOrDie(), + }, + }) + } return controllerRoles, controllerRoleBindings } diff --git a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go index 1ab74d57a77..d74ca2f4946 100644 --- a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go +++ b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go @@ -131,8 +131,8 @@ type KubeControllerManagerConfiguration struct { JobController JobControllerConfiguration // CronJobControllerConfiguration holds configuration for CronJobController related features. CronJobController CronJobControllerConfiguration - // NamespaceControllerConfiguration holds configuration for NamespaceController - // related features. + // LegacySATokenCleanerConfiguration holds configuration for LegacySATokenCleaner related features. + LegacySATokenCleaner LegacySATokenCleanerConfiguration // NamespaceControllerConfiguration holds configuration for NamespaceController // related features. NamespaceController NamespaceControllerConfiguration @@ -357,6 +357,13 @@ type CronJobControllerConfiguration struct { ConcurrentCronJobSyncs int32 } +// LegacySATokenCleanerConfiguration contains elements describing LegacySATokenCleaner +type LegacySATokenCleanerConfiguration struct { + // CleanUpPeriod is the period of time since the last usage of an + // auto-generated service account token before it can be deleted. + CleanUpPeriod metav1.Duration +} + // NamespaceControllerConfiguration contains elements describing NamespaceController. type NamespaceControllerConfiguration struct { // namespaceSyncPeriod is the period for syncing namespace life-cycle diff --git a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/zz_generated.deepcopy.go index 441e3bb2f75..7a457692277 100644 --- a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/zz_generated.deepcopy.go @@ -310,6 +310,7 @@ func (in *KubeControllerManagerConfiguration) DeepCopyInto(out *KubeControllerMa out.HPAController = in.HPAController out.JobController = in.JobController out.CronJobController = in.CronJobController + out.LegacySATokenCleaner = in.LegacySATokenCleaner out.NamespaceController = in.NamespaceController out.NodeIPAMController = in.NodeIPAMController out.NodeLifecycleController = in.NodeLifecycleController @@ -342,6 +343,23 @@ func (in *KubeControllerManagerConfiguration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LegacySATokenCleanerConfiguration) DeepCopyInto(out *LegacySATokenCleanerConfiguration) { + *out = *in + out.CleanUpPeriod = in.CleanUpPeriod + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LegacySATokenCleanerConfiguration. +func (in *LegacySATokenCleanerConfiguration) DeepCopy() *LegacySATokenCleanerConfiguration { + if in == nil { + return nil + } + out := new(LegacySATokenCleanerConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NamespaceControllerConfiguration) DeepCopyInto(out *NamespaceControllerConfiguration) { *out = *in diff --git a/test/integration/serviceaccount/legacy_service_account_token_clean_up_test.go b/test/integration/serviceaccount/legacy_service_account_token_clean_up_test.go new file mode 100644 index 00000000000..5f59819a93f --- /dev/null +++ b/test/integration/serviceaccount/legacy_service_account_token_clean_up_test.go @@ -0,0 +1,251 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serviceaccount + +// This file tests the legacy service account token cleaning-up. + +import ( + "context" + "fmt" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" + clientinformers "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + listersv1 "k8s.io/client-go/listers/core/v1" + featuregatetesting "k8s.io/component-base/featuregate/testing" + serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" + "k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking" + kubefeatures "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/serviceaccount" + "k8s.io/utils/clock" + testingclock "k8s.io/utils/clock/testing" +) + +const ( + dateFormat = "2006-01-02" + cleanUpPeriod = 24 * time.Hour + syncInterval = 1 * time.Second +) + +func TestLegacyServiceAccountTokenCleanUp(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, kubefeatures.LegacyServiceAccountTokenCleanUp, true)() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fakeClock := testingclock.NewFakeClock(time.Now().UTC()) + + c, config, stopFunc, informers, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) + defer stopFunc() + if err != nil { + t.Fatalf("failed to setup ServiceAccounts server: %v", err) + } + + // start legacy service account token cleaner + startLegacyServiceAccountTokenCleaner(ctx, c, fakeClock, informers) + + // wait configmap to label with tracking date + if err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { + configMap, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, legacytokentracking.ConfigMapName, metav1.GetOptions{}) + if err != nil { + return false, err + } + _, exist := configMap.Data[legacytokentracking.ConfigMapDataKey] + if !exist { + return false, fmt.Errorf("configMap does not have since label") + } + return true, nil + }); err != nil { + t.Fatalf("failed to wait configmap starts to track: %v", err) + } + + // create service account + myns := "clean-ns" + _, err = c.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: myns}}, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + t.Fatalf("could not create namespace: %v", err) + } + mysa, err := c.CoreV1().ServiceAccounts(myns).Create(context.TODO(), &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: readOnlyServiceAccountName}}, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Service Account not created: %v", err) + } + + tests := []struct { + name string + secretName string + secretTokenData string + expectCleanedUp bool + lastUsedLabel bool + isPodMounted bool + isManual bool + }{ + { + name: "auto created legacy token without pod binding", + secretName: "auto-token-without-pod-mounting-a", + lastUsedLabel: true, + isManual: false, + isPodMounted: false, + expectCleanedUp: true, + }, + { + name: "manually created legacy token", + secretName: "manual-token", + lastUsedLabel: true, + isManual: true, + isPodMounted: false, + expectCleanedUp: false, + }, + { + name: "auto created legacy token with pod binding", + secretName: "auto-token-with-pod-mounting", + lastUsedLabel: true, + isManual: false, + isPodMounted: true, + expectCleanedUp: false, + }, + { + name: "auto created legacy token without pod binding, secret has not been used after tracking", + secretName: "auto-token-without-pod-mounting-b", + lastUsedLabel: false, + isManual: false, + isPodMounted: false, + expectCleanedUp: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // create secret + secret, err := createServiceAccountToken(c, mysa, myns, test.secretName) + if err != nil { + t.Fatalf("Secret not created: %v", err) + } + if !test.isManual { + if err := addReferencedServiceAccountToken(c, myns, readOnlyServiceAccountName, secret); err != nil { + t.Fatal(err) + } + } + podLister := informers.Core().V1().Pods().Lister() + if test.isPodMounted { + _, err = createAutotokenMountedPod(c, myns, test.secretName, podLister) + if err != nil { + t.Fatalf("Pod not created: %v", err) + } + } + + myConfig := *config + wh := &warningHandler{} + myConfig.WarningHandler = wh + myConfig.BearerToken = string(string(secret.Data[v1.ServiceAccountTokenKey])) + roClient := clientset.NewForConfigOrDie(&myConfig) + + // the secret should not be labeled with LastUsedLabelKey. + liveSecret, err := c.CoreV1().Secrets(myns).Get(context.TODO(), test.secretName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Could not get secret: %v", err) + } + _, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey] + if ok { + t.Fatalf("Secret %s should not have the lastUsed label", test.secretName) + } + + // authenticate legacy tokens + if test.lastUsedLabel { + doServiceAccountAPIRequests(t, roClient, myns, true, true, false) + // all service account tokens should be labeled with LastUsedLabelKey. + liveSecret, err = c.CoreV1().Secrets(myns).Get(context.TODO(), test.secretName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Could not get secret: %v", err) + } + lastUsed, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey] + if !ok { + t.Fatalf("The secret %s should be labeled lastUsed time: %s", test.secretName, lastUsed) + } else { + t.Logf("The secret %s has been labeled with %s", test.secretName, lastUsed) + } + } + + _, err = c.CoreV1().Secrets(myns).Get(context.TODO(), test.secretName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + t.Fatalf("The secret %s should not be cleaned up, err: %v", test.secretName, err) + } else { + t.Fatalf("Failed to get secret %s, err: %v", test.secretName, err) + } + } + + fakeClock.Step(cleanUpPeriod + 24*time.Hour) + time.Sleep(2 * syncInterval) + liveSecret, err = c.CoreV1().Secrets(myns).Get(context.TODO(), test.secretName, metav1.GetOptions{}) + if test.expectCleanedUp && err == nil { + t.Fatalf("The secret %s should be cleaned up. time: %v; creationTime: %v", test.secretName, fakeClock.Now().UTC(), liveSecret.CreationTimestamp) + } else if !test.expectCleanedUp && err != nil { + if apierrors.IsNotFound(err) { + t.Fatalf("The secret %s should not be cleaned up, err: %v", test.secretName, err) + } else { + t.Fatalf("Failed to get secret %s, err: %v", test.secretName, err) + } + } + }) + } +} + +func startLegacyServiceAccountTokenCleaner(ctx context.Context, client clientset.Interface, fakeClock clock.Clock, informers clientinformers.SharedInformerFactory) { + legacySATokenCleaner, _ := serviceaccountcontroller.NewLegacySATokenCleaner( + informers.Core().V1().ServiceAccounts(), + informers.Core().V1().Secrets(), + informers.Core().V1().Pods(), + client, + fakeClock, + serviceaccountcontroller.LegacySATokenCleanerOptions{ + SyncInterval: syncInterval, + CleanUpPeriod: cleanUpPeriod, + }) + go legacySATokenCleaner.Run(ctx) + informers.Start(ctx.Done()) +} + +func createAutotokenMountedPod(c clientset.Interface, ns, secretName string, podLister listersv1.PodLister) (*v1.Pod, error) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "token-bound-pod", + Namespace: ns, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + {Name: "name", Image: "image"}, + }, + Volumes: []v1.Volume{{Name: "foo", VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: secretName}}}}, + }, + } + pod, err := c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create pod with token (%s:%s) bound, err: %v", ns, secretName, err) + } + err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { + pod, err = podLister.Pods(ns).Get("token-bound-pod") + if err != nil { + return false, fmt.Errorf("failed to get pod with token (%s:%s) bound, err: %v", ns, secretName, err) + } + return true, nil + }) + return pod, nil +} diff --git a/test/integration/serviceaccount/service_account_test.go b/test/integration/serviceaccount/service_account_test.go index 7e6d2c18eec..13c8f160573 100644 --- a/test/integration/serviceaccount/service_account_test.go +++ b/test/integration/serviceaccount/service_account_test.go @@ -52,8 +52,6 @@ import ( const ( readOnlyServiceAccountName = "ro" readWriteServiceAccountName = "rw" - - dateFormat = "2006-01-02" ) func TestServiceAccountAutoCreate(t *testing.T) { @@ -61,7 +59,7 @@ func TestServiceAccountAutoCreate(t *testing.T) { ctx, cancel := context.WithCancel(ctx) defer cancel() - c, _, stopFunc, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) + c, _, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) defer stopFunc() if err != nil { t.Fatalf("failed to setup ServiceAccounts server: %v", err) @@ -102,7 +100,7 @@ func TestServiceAccountTokenAutoMount(t *testing.T) { ctx, cancel := context.WithCancel(ctx) defer cancel() - c, _, stopFunc, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) + c, _, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) defer stopFunc() if err != nil { t.Fatalf("failed to setup ServiceAccounts server: %v", err) @@ -148,7 +146,7 @@ func TestServiceAccountTokenAuthentication(t *testing.T) { ctx, cancel := context.WithCancel(ctx) defer cancel() - c, config, stopFunc, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) + c, config, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) defer stopFunc() if err != nil { t.Fatalf("failed to setup ServiceAccounts server: %v", err) @@ -229,7 +227,7 @@ func TestLegacyServiceAccountTokenTracking(t *testing.T) { ctx, cancel := context.WithCancel(ctx) defer cancel() - c, config, stopFunc, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) + c, config, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(ctx, t) defer stopFunc() if err != nil { t.Fatalf("failed to setup ServiceAccounts server: %v", err) @@ -347,7 +345,7 @@ func TestLegacyServiceAccountTokenTracking(t *testing.T) { // startServiceAccountTestServerAndWaitForCaches returns a started server // It is the responsibility of the caller to ensure the returned stopFunc is called -func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testing.T) (clientset.Interface, *restclient.Config, func(), error) { +func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testing.T) (clientset.Interface, *restclient.Config, func(), clientinformers.SharedInformerFactory, error) { var serviceAccountKey interface{} ctx, cancel := context.WithCancel(ctx) @@ -404,7 +402,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi // Start the service account and service account token controllers tokenGenerator, err := serviceaccount.JWTTokenGenerator(serviceaccount.LegacyIssuer, serviceAccountKey) if err != nil { - return rootClientset, clientConfig, stop, err + return rootClientset, clientConfig, stop, informers, err } tokenController, err := serviceaccountcontroller.NewTokensController( informers.Core().V1().ServiceAccounts(), @@ -415,7 +413,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi }, ) if err != nil { - return rootClientset, clientConfig, stop, err + return rootClientset, clientConfig, stop, informers, err } go tokenController.Run(ctx, 1) @@ -426,7 +424,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi serviceaccountcontroller.DefaultServiceAccountsControllerOptions(), ) if err != nil { - return rootClientset, clientConfig, stop, err + return rootClientset, clientConfig, stop, informers, err } informers.Start(ctx.Done()) go serviceAccountController.Run(ctx, 5) @@ -437,7 +435,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi // thus we wait until caches have synced informers.WaitForCacheSync(ctx.Done()) - return rootClientset, clientConfig, stop, nil + return rootClientset, clientConfig, stop, informers, nil } func getServiceAccount(c clientset.Interface, ns string, name string, shouldWait bool) (*v1.ServiceAccount, error) {