diff --git a/pkg/credentialprovider/plugin/config.go b/pkg/credentialprovider/plugin/config.go index a3d03aa83cb..1f6f441c7e7 100644 --- a/pkg/credentialprovider/plugin/config.go +++ b/pkg/credentialprovider/plugin/config.go @@ -22,7 +22,9 @@ import ( "strings" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" "k8s.io/kubernetes/pkg/credentialprovider" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" ) @@ -70,7 +72,7 @@ func decode(data []byte) (*kubeletconfig.CredentialProviderConfig, error) { } // validateCredentialProviderConfig validates CredentialProviderConfig. -func validateCredentialProviderConfig(config *kubeletconfig.CredentialProviderConfig) field.ErrorList { +func validateCredentialProviderConfig(config *kubeletconfig.CredentialProviderConfig, saTokenForCredentialProviders bool) field.ErrorList { allErrs := field.ErrorList{} if len(config.Providers) == 0 { @@ -125,7 +127,56 @@ func validateCredentialProviderConfig(config *kubeletconfig.CredentialProviderCo if provider.DefaultCacheDuration != nil && provider.DefaultCacheDuration.Duration < 0 { allErrs = append(allErrs, field.Invalid(fieldPath.Child("defaultCacheDuration"), provider.DefaultCacheDuration.Duration, "defaultCacheDuration must be greater than or equal to 0")) } + + if provider.TokenAttributes != nil { + fldPath := fieldPath.Child("tokenAttributes") + if !saTokenForCredentialProviders { + allErrs = append(allErrs, field.Forbidden(fldPath, "tokenAttributes is not supported when KubeletServiceAccountTokenForCredentialProviders feature gate is disabled")) + } + if len(provider.TokenAttributes.ServiceAccountTokenAudience) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("serviceAccountTokenAudience"), "serviceAccountTokenAudience is required")) + } + if provider.TokenAttributes.RequireServiceAccount == nil { + allErrs = append(allErrs, field.Required(fldPath.Child("requireServiceAccount"), "requireServiceAccount is required")) + } + if provider.APIVersion != credentialproviderv1.SchemeGroupVersion.String() { + allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("tokenAttributes is only supported for %s API version", credentialproviderv1.SchemeGroupVersion.String()))) + } + + if provider.TokenAttributes.RequireServiceAccount != nil && !*provider.TokenAttributes.RequireServiceAccount && len(provider.TokenAttributes.RequiredServiceAccountAnnotationKeys) > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("requiredServiceAccountAnnotationKeys"), "requireServiceAccount cannot be false when requiredServiceAccountAnnotationKeys is set")) + } + + allErrs = append(allErrs, validateServiceAccountAnnotationKeys(fldPath.Child("requiredServiceAccountAnnotationKeys"), provider.TokenAttributes.RequiredServiceAccountAnnotationKeys)...) + allErrs = append(allErrs, validateServiceAccountAnnotationKeys(fldPath.Child("optionalServiceAccountAnnotationKeys"), provider.TokenAttributes.OptionalServiceAccountAnnotationKeys)...) + + requiredServiceAccountAnnotationKeys := sets.New[string](provider.TokenAttributes.RequiredServiceAccountAnnotationKeys...) + optionalServiceAccountAnnotationKeys := sets.New[string](provider.TokenAttributes.OptionalServiceAccountAnnotationKeys...) + duplicateAnnotationKeys := requiredServiceAccountAnnotationKeys.Intersection(optionalServiceAccountAnnotationKeys) + if duplicateAnnotationKeys.Len() > 0 { + allErrs = append(allErrs, field.Invalid(fldPath, sets.List(duplicateAnnotationKeys), "annotation keys cannot be both required and optional")) + } + } } return allErrs } + +// validateServiceAccountAnnotationKeys validates the service account annotation keys. +func validateServiceAccountAnnotationKeys(fldPath *field.Path, keys []string) field.ErrorList { + allErrs := field.ErrorList{} + + seenAnnotationKeys := sets.New[string]() + // Using the validation logic for keys from https://github.com/kubernetes/kubernetes/blob/69dbc74417304328a9fd3c161643dc4f0a057f41/staging/src/k8s.io/apimachinery/pkg/api/validation/objectmeta.go#L46-L51 + for _, k := range keys { + // The rule is QualifiedName except that case doesn't matter, so convert to lowercase before checking. + for _, msg := range validation.IsQualifiedName(strings.ToLower(k)) { + allErrs = append(allErrs, field.Invalid(fldPath, k, msg)) + } + if seenAnnotationKeys.Has(k) { + allErrs = append(allErrs, field.Duplicate(fldPath, k)) + } + seenAnnotationKeys.Insert(k) + } + return allErrs +} diff --git a/pkg/credentialprovider/plugin/config_test.go b/pkg/credentialprovider/plugin/config_test.go index 8583964763c..c0cefc65183 100644 --- a/pkg/credentialprovider/plugin/config_test.go +++ b/pkg/credentialprovider/plugin/config_test.go @@ -19,16 +19,17 @@ package plugin import ( "os" "reflect" + "strings" "testing" "time" "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/errors" utiltesting "k8s.io/client-go/util/testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/utils/ptr" ) func Test_readCredentialProviderConfigFile(t *testing.T) { @@ -337,6 +338,48 @@ providers: config: nil, expectErr: `strict decoding error: unknown field "providers[0].unknownField"`, }, + { + name: "v1alpha1 config with token attributes should fail", + configData: `--- +kind: CredentialProviderConfig +apiVersion: kubelet.config.k8s.io/v1alpha1 +providers: + - name: test + matchImages: + - "registry.io/foobar" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + tokenAttributes: + serviceAccountTokenAudience: audience + args: + - --v=5 + env: + - name: FOO + value: BAR`, + config: nil, + expectErr: `strict decoding error: unknown field "providers[0].tokenAttributes"`, + }, + { + name: "v1beta1 config with token attributes should fail", + configData: `--- +kind: CredentialProviderConfig +apiVersion: kubelet.config.k8s.io/v1beta1 +providers: + - name: test + matchImages: + - "registry.io/foobar" + defaultCacheDuration: 10m + apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 + tokenAttributes: + serviceAccountTokenAudience: audience + args: + - --v=5 + env: + - name: FOO + value: BAR`, + config: nil, + expectErr: `strict decoding error: unknown field "providers[0].tokenAttributes"`, + }, } for _, testcase := range testcases { @@ -347,17 +390,19 @@ providers: } defer utiltesting.CloseAndRemove(t, file) - _, err = file.WriteString(testcase.configData) - if err != nil { + if _, err = file.WriteString(testcase.configData); err != nil { t.Fatal(err) } authConfig, err := readCredentialProviderConfigFile(file.Name()) - if err != nil && len(testcase.expectErr) == 0 { - t.Fatal(err) - } - - if err == nil && len(testcase.expectErr) > 0 { + if err != nil { + if len(testcase.expectErr) == 0 { + t.Fatal(err) + } + if !strings.Contains(err.Error(), testcase.expectErr) { + t.Fatalf("expected error %q but got %q", testcase.expectErr, err.Error()) + } + } else if len(testcase.expectErr) > 0 { t.Fatalf("expected error %q but got none", testcase.expectErr) } @@ -554,11 +599,248 @@ func Test_validateCredentialProviderConfig(t *testing.T) { }, }, }, + { + name: "token attributes set without KubeletServiceAccountTokenForCredentialProviders feature gate enabled", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + }, + }, + }, + }, + expectErr: `providers.tokenAttributes: Forbidden: tokenAttributes is not supported when KubeletServiceAccountTokenForCredentialProviders feature gate is disabled`, + }, + { + name: "token attributes not nil but empty ServiceAccountTokenAudience", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + RequiredServiceAccountAnnotationKeys: []string{"prefix.io/annotation-1", "prefix.io/annotation-2"}, + RequireServiceAccount: ptr.To(true), + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes.serviceAccountTokenAudience: Required value: serviceAccountTokenAudience is required`, + }, + { + name: "token attributes not nil but empty ServiceAccountTokenRequired", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequiredServiceAccountAnnotationKeys: []string{"prefix.io/annotation-1", "prefix.io/annotation-2"}, + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes.requireServiceAccount: Required value: requireServiceAccount is required`, + }, + { + name: "required service account annotation keys not qualified name (same validation as metav1.ObjectMeta)", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + RequiredServiceAccountAnnotationKeys: []string{"cantendwithadash-", "now-with-dashes/simple"}, // first key is invalid + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes.requiredServiceAccountAnnotationKeys: Invalid value: "cantendwithadash-": name part must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or '123-abc', regex used for validation is '([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]')`, + }, + { + name: "optional service account annotation keys not qualified name (same validation as metav1.ObjectMeta)", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + OptionalServiceAccountAnnotationKeys: []string{"cantendwithadash-", "now-with-dashes/simple"}, // first key is invalid + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes.optionalServiceAccountAnnotationKeys: Invalid value: "cantendwithadash-": name part must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or '123-abc', regex used for validation is '([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]')`, + }, + { + name: "duplicate required service account annotation keys", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + RequiredServiceAccountAnnotationKeys: []string{"now-with-dashes/simple", "now-with-dashes/simple"}, + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes.requiredServiceAccountAnnotationKeys: Duplicate value: "now-with-dashes/simple"`, + }, + { + name: "duplicate optional service account annotation keys", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + OptionalServiceAccountAnnotationKeys: []string{"now-with-dashes/simple", "now-with-dashes/simple"}, + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes.optionalServiceAccountAnnotationKeys: Duplicate value: "now-with-dashes/simple"`, + }, + { + name: "annotation key in required and optional keys", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + RequiredServiceAccountAnnotationKeys: []string{"now-with-dashes/simple-1", "now-with-dashes/simple-2"}, + OptionalServiceAccountAnnotationKeys: []string{"now-with-dashes/simple-2", "now-with-dashes/simple-3"}, + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes: Invalid value: []string{"now-with-dashes/simple-2"}: annotation keys cannot be both required and optional`, + }, + { + name: "required annotation keys set when requireServiceAccount is false", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(false), + RequiredServiceAccountAnnotationKeys: []string{"now-with-dashes/simple-1", "now-with-dashes/simple-2"}, + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes.requiredServiceAccountAnnotationKeys: Forbidden: requireServiceAccount cannot be false when requiredServiceAccountAnnotationKeys is set`, + }, + { + name: "valid config with KubeletServiceAccountTokenForCredentialProviders feature gate enabled", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + RequiredServiceAccountAnnotationKeys: []string{"now-with-dashes/simple-1", "now-with-dashes/simple-2"}, + OptionalServiceAccountAnnotationKeys: []string{"now-with-dashes/simple-3"}, + }, + }, + }, + }, + saTokenForCredentialProviders: true, + }, + { + name: "tokenAttributes set with credentialprovider.kubelet.k8s.io/v1alpha1 APIVersion", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1alpha1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + RequiredServiceAccountAnnotationKeys: []string{"now-with-dashes/simple"}, + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes: Forbidden: tokenAttributes is only supported for credentialprovider.kubelet.k8s.io/v1 API version`, + }, + { + name: "tokenAttributes set with credentialprovider.kubelet.k8s.io/v1beta1 APIVersion", + config: &kubeletconfig.CredentialProviderConfig{ + Providers: []kubeletconfig.CredentialProvider{ + { + Name: "foobar", + MatchImages: []string{"foobar.registry.io"}, + DefaultCacheDuration: &metav1.Duration{Duration: time.Minute}, + APIVersion: "credentialprovider.kubelet.k8s.io/v1beta1", + TokenAttributes: &kubeletconfig.ServiceAccountTokenAttributes{ + ServiceAccountTokenAudience: "audience", + RequireServiceAccount: ptr.To(true), + RequiredServiceAccountAnnotationKeys: []string{"now-with-dashes/simple"}, + }, + }, + }, + }, + saTokenForCredentialProviders: true, + expectErr: `providers.tokenAttributes: Forbidden: tokenAttributes is only supported for credentialprovider.kubelet.k8s.io/v1 API version`, + }, } for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { - errs := validateCredentialProviderConfig(testcase.config).ToAggregate() + errs := validateCredentialProviderConfig(testcase.config, testcase.saTokenForCredentialProviders).ToAggregate() if d := cmp.Diff(testcase.expectErr, errString(errs)); d != "" { t.Fatalf("CredentialProviderConfig validation mismatch (-want +got):\n%s", d) } diff --git a/pkg/credentialprovider/plugin/plugin.go b/pkg/credentialprovider/plugin/plugin.go index e0ec9a41661..2a6ce7b9441 100644 --- a/pkg/credentialprovider/plugin/plugin.go +++ b/pkg/credentialprovider/plugin/plugin.go @@ -19,6 +19,7 @@ package plugin import ( "bytes" "context" + "crypto/sha256" "errors" "fmt" "os" @@ -28,12 +29,18 @@ import ( "sync" "time" + "golang.org/x/crypto/cryptobyte" "golang.org/x/sync/singleflight" + authenticationv1 "k8s.io/api/authentication/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" credentialproviderapi "k8s.io/kubelet/pkg/apis/credentialprovider" @@ -42,6 +49,7 @@ import ( credentialproviderv1alpha1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1" credentialproviderv1beta1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1beta1" "k8s.io/kubernetes/pkg/credentialprovider" + "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" kubeletconfigv1 "k8s.io/kubernetes/pkg/kubelet/apis/config/v1" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/config/v1alpha1" @@ -75,7 +83,10 @@ func init() { // RegisterCredentialProviderPlugins is called from kubelet to register external credential provider // plugins according to the CredentialProviderConfig config file. -func RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir string) error { +func RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir string, + getServiceAccountToken func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error), + getServiceAccount func(namespace, name string) (*v1.ServiceAccount, error), +) error { if _, err := os.Stat(pluginBinDir); err != nil { if os.IsNotExist(err) { return fmt.Errorf("plugin binary directory %s did not exist", pluginBinDir) @@ -89,8 +100,8 @@ func RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir string) er return err } - errs := validateCredentialProviderConfig(credentialProviderConfig) - if len(errs) > 0 { + saTokenForCredentialProvidersFeatureEnabled := utilfeature.DefaultFeatureGate.Enabled(features.KubeletServiceAccountTokenForCredentialProviders) + if errs := validateCredentialProviderConfig(credentialProviderConfig, saTokenForCredentialProvidersFeatureEnabled); len(errs) > 0 { return fmt.Errorf("failed to validate credential provider config: %v", errs.ToAggregate()) } @@ -109,19 +120,22 @@ func RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir string) er return fmt.Errorf("error inspecting binary executable %s: %w", pluginBin, err) } - plugin, err := newPluginProvider(pluginBinDir, provider) + plugin, err := newPluginProvider(pluginBinDir, provider, getServiceAccountToken, getServiceAccount) if err != nil { return fmt.Errorf("error initializing plugin provider %s: %w", provider.Name, err) } - credentialprovider.RegisterCredentialProvider(provider.Name, plugin) + registerCredentialProviderPlugin(provider.Name, plugin) } return nil } // newPluginProvider returns a new pluginProvider based on the credential provider config. -func newPluginProvider(pluginBinDir string, provider kubeletconfig.CredentialProvider) (*pluginProvider, error) { +func newPluginProvider(pluginBinDir string, provider kubeletconfig.CredentialProvider, + getServiceAccountToken func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error), + getServiceAccount func(namespace, name string) (*v1.ServiceAccount, error), +) (*pluginProvider, error) { mediaType := "application/json" info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) if !ok { @@ -134,7 +148,6 @@ func newPluginProvider(pluginBinDir string, provider kubeletconfig.CredentialPro } clock := clock.RealClock{} - return &pluginProvider{ clock: clock, matchImages: provider.MatchImages, @@ -150,6 +163,7 @@ func newPluginProvider(pluginBinDir string, provider kubeletconfig.CredentialPro envVars: provider.Env, environ: os.Environ, }, + serviceAccountProvider: newServiceAccountProvider(provider, getServiceAccount, getServiceAccountToken), }, nil } @@ -178,6 +192,101 @@ type pluginProvider struct { // lastCachePurge is the last time cache is cleaned for expired entries. lastCachePurge time.Time + + // serviceAccountProvider holds the logic for handling service account tokens when needed. + serviceAccountProvider *serviceAccountProvider +} + +type serviceAccountProvider struct { + audience string + requireServiceAccount bool + getServiceAccountFunc func(namespace, name string) (*v1.ServiceAccount, error) + getServiceAccountTokenFunc func(podNamespace, serviceAccountName string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) + requiredServiceAccountAnnotationKeys []string + optionalServiceAccountAnnotationKeys []string +} + +func newServiceAccountProvider( + provider kubeletconfig.CredentialProvider, + getServiceAccount func(namespace, name string) (*v1.ServiceAccount, error), + getServiceAccountToken func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error), +) *serviceAccountProvider { + featureGateEnabled := utilfeature.DefaultFeatureGate.Enabled(features.KubeletServiceAccountTokenForCredentialProviders) + serviceAccountTokenAudienceSet := provider.TokenAttributes != nil && len(provider.TokenAttributes.ServiceAccountTokenAudience) > 0 + + if !featureGateEnabled || !serviceAccountTokenAudienceSet { + return nil + } + + return &serviceAccountProvider{ + audience: provider.TokenAttributes.ServiceAccountTokenAudience, + requireServiceAccount: *provider.TokenAttributes.RequireServiceAccount, + getServiceAccountFunc: getServiceAccount, + getServiceAccountTokenFunc: getServiceAccountToken, + requiredServiceAccountAnnotationKeys: provider.TokenAttributes.RequiredServiceAccountAnnotationKeys, + optionalServiceAccountAnnotationKeys: provider.TokenAttributes.OptionalServiceAccountAnnotationKeys, + } +} + +type requiredAnnotationNotFoundError string + +func (e requiredAnnotationNotFoundError) Error() string { + return fmt.Sprintf("required annotation %s not found", string(e)) +} + +// getServiceAccountData returns the service account UID and required annotations for the service account. +// If the service account does not exist, an error is returned. +// saAnnotations is a map of annotation keys and values that the plugin requires to generate credentials +// that's defined in the tokenAttributes in the credential provider config. +// requiredServiceAccountAnnotationKeys are the keys that are required to be present in the service account. +// If any of the keys defined in this list are not present in the service account, kubelet will not invoke the plugin +// and will return an error. +// optionalServiceAccountAnnotationKeys are the keys that are optional to be present in the service account. +// If present, they will be added to the saAnnotations map. +func (s *serviceAccountProvider) getServiceAccountData(namespace, name string) (types.UID, map[string]string, error) { + sa, err := s.getServiceAccountFunc(namespace, name) + if err != nil { + return "", nil, err + } + + saAnnotations := make(map[string]string, len(s.requiredServiceAccountAnnotationKeys)+len(s.optionalServiceAccountAnnotationKeys)) + for _, k := range s.requiredServiceAccountAnnotationKeys { + val, ok := sa.Annotations[k] + if !ok { + return "", nil, requiredAnnotationNotFoundError(k) + } + saAnnotations[k] = val + } + + for _, k := range s.optionalServiceAccountAnnotationKeys { + if val, ok := sa.Annotations[k]; ok { + saAnnotations[k] = val + } + } + + return sa.UID, saAnnotations, nil +} + +// getServiceAccountToken returns a service account token for the service account. +func (s *serviceAccountProvider) getServiceAccountToken(podNamespace, podName, serviceAccountName string, podUID types.UID) (string, error) { + tr, err := s.getServiceAccountTokenFunc(podNamespace, serviceAccountName, &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + Audiences: []string{s.audience}, + // expirationSeconds is not set explicitly here. It has the same default value of "ExpirationSeconds" in the TokenRequestSpec. + BoundObjectRef: &authenticationv1.BoundObjectReference{ + APIVersion: "v1", + Kind: "Pod", + Name: podName, + UID: podUID, + }, + }, + }) + + if err != nil { + return "", err + } + + return tr.Status.Token, nil } // cacheEntry is the cache object that will be stored in cache.Store. @@ -204,15 +313,86 @@ func (c *cacheExpirationPolicy) IsExpired(entry *cache.TimestampedEntry) bool { return c.clock.Now().After(entry.Obj.(*cacheEntry).expiresAt) } -// Provide returns a credentialprovider.DockerConfig based on the credentials returned +// perPluginProvider holds the shared pluginProvider and the per-request information +// like podName, podNamespace, podUID and serviceAccountName. +// This is used to provide the per-request information to the pluginProvider.provide method, so +// that the plugin can use this information to get the pod's service account and generate bound service account tokens +// for plugins running in service account token mode. +type perPodPluginProvider struct { + name string + + provider *pluginProvider + + podNamespace string + podName string + podUID types.UID + + serviceAccountName string +} + +// Enabled always returns true since registration of the plugin via kubelet implies it should be enabled. +func (p *perPodPluginProvider) Enabled() bool { + return true +} + +func (p *perPodPluginProvider) Provide(image string) credentialprovider.DockerConfig { + return p.provider.provide(image, p.podNamespace, p.podName, p.podUID, p.serviceAccountName) +} + +// provide returns a credentialprovider.DockerConfig based on the credentials returned // from cache or the exec plugin. -func (p *pluginProvider) Provide(image string) credentialprovider.DockerConfig { +func (p *pluginProvider) provide(image, podNamespace, podName string, podUID types.UID, serviceAccountName string) credentialprovider.DockerConfig { if !p.isImageAllowed(image) { return credentialprovider.DockerConfig{} } - cachedConfig, found, err := p.getCachedCredentials(image) - if err != nil { + var serviceAccountUID types.UID + var serviceAccountToken string + var saAnnotations map[string]string + var err error + var serviceAccountCacheKey string + + if p.serviceAccountProvider != nil { + if len(serviceAccountName) == 0 && p.serviceAccountProvider.requireServiceAccount { + klog.V(5).Infof("Service account name is empty for pod %s/%s", podNamespace, podName) + return credentialprovider.DockerConfig{} + } + + // If the service account name is empty and the plugin has indicated that invoking the plugin + // without a service account is allowed, we will continue without generating a service account token. + // This is useful for plugins that are running in service account token mode and are also used + // to pull images for pods without service accounts (e.g., static pods). + if len(serviceAccountName) > 0 { + if serviceAccountUID, saAnnotations, err = p.serviceAccountProvider.getServiceAccountData(podNamespace, serviceAccountName); err != nil { + var requiredAnnotationNotFoundErr requiredAnnotationNotFoundError + if errors.As(err, &requiredAnnotationNotFoundErr) { + // The required annotation could be a mechanism for individual workloads to opt in to using service account tokens + // for image pull. If any of the required annotation is missing, we will not invoke the plugin. We will log the error + // at higher verbosity level as it could be noisy. + klog.V(5).Infof("Failed to get service account data %s/%s: %v", podNamespace, serviceAccountName, err) + return credentialprovider.DockerConfig{} + } + + klog.Errorf("Failed to get service account %s/%s: %v", podNamespace, serviceAccountName, err) + return credentialprovider.DockerConfig{} + } + + if serviceAccountToken, err = p.serviceAccountProvider.getServiceAccountToken(podNamespace, podName, serviceAccountName, podUID); err != nil { + klog.Errorf("Error getting service account token %s/%s: %v", podNamespace, serviceAccountName, err) + return credentialprovider.DockerConfig{} + } + + serviceAccountCacheKey, err = generateServiceAccountCacheKey(podNamespace, serviceAccountName, serviceAccountUID, saAnnotations) + if err != nil { + klog.Errorf("Error generating service account cache key: %v", err) + return credentialprovider.DockerConfig{} + } + } + } + + // Check if the credentials are cached and return them if found. + cachedConfig, found, errCache := p.getCachedCredentials(image, serviceAccountCacheKey) + if errCache != nil { klog.Errorf("Failed to get cached docker config: %v", err) return credentialprovider.DockerConfig{} } @@ -227,8 +407,23 @@ func (p *pluginProvider) Provide(image string) credentialprovider.DockerConfig { // foo.bar.registry // foo.bar.registry/image1 // foo.bar.registry/image2 - res, err, _ := p.group.Do(image, func() (interface{}, error) { - return p.plugin.ExecPlugin(context.Background(), image) + // When the plugin is operating in the service account token mode, the singleflight key is the image plus the serviceAccountCacheKey + // which is generated from the service account namespace, name, uid and the annotations passed to the plugin. + singleFlightKey := image + if p.serviceAccountProvider != nil && len(serviceAccountName) > 0 { + // When the plugin is operating in the service account token mode, the singleflight key is the + // image + sa annotations + sa token. + // This does mean the singleflight key is different for each image pull request (even if the image is the same) + // and the workload is using the same service account. + // In the future, when we support caching of the service account token for pod-sa pairs, this will be singleflighted + // for different containers in the same pod using the same image. + if singleFlightKey, err = generateSingleFlightKey(image, getHashIfNotEmpty(serviceAccountToken), saAnnotations); err != nil { + klog.Errorf("Error generating singleflight key: %v", err) + return credentialprovider.DockerConfig{} + } + } + res, err, _ := p.group.Do(singleFlightKey, func() (interface{}, error) { + return p.plugin.ExecPlugin(context.Background(), image, serviceAccountToken, saAnnotations) }) if err != nil { @@ -280,6 +475,12 @@ func (p *pluginProvider) Provide(image string) credentialprovider.DockerConfig { expiresAt = p.clock.Now().Add(response.CacheDuration.Duration) } + cacheKey, err = generateCacheKey(cacheKey, serviceAccountCacheKey) + if err != nil { + klog.Errorf("Error generating cache key: %v", err) + return credentialprovider.DockerConfig{} + } + cachedEntry := &cacheEntry{ key: cacheKey, credentials: dockerConfig, @@ -310,7 +511,7 @@ func (p *pluginProvider) isImageAllowed(image string) bool { } // getCachedCredentials returns a credentialprovider.DockerConfig if cached from the plugin. -func (p *pluginProvider) getCachedCredentials(image string) (credentialprovider.DockerConfig, bool, error) { +func (p *pluginProvider) getCachedCredentials(image, serviceAccountCacheKey string) (credentialprovider.DockerConfig, bool, error) { p.Lock() if p.clock.Now().After(p.lastCachePurge.Add(cachePurgeInterval)) { // NewExpirationCache purges expired entries when List() is called @@ -321,7 +522,12 @@ func (p *pluginProvider) getCachedCredentials(image string) (credentialprovider. } p.Unlock() - obj, found, err := p.cache.GetByKey(image) + cacheKey, err := generateCacheKey(image, serviceAccountCacheKey) + if err != nil { + return nil, false, fmt.Errorf("error generating cache key: %w", err) + } + + obj, found, err := p.cache.GetByKey(cacheKey) if err != nil { return nil, false, err } @@ -331,7 +537,13 @@ func (p *pluginProvider) getCachedCredentials(image string) (credentialprovider. } registry := parseRegistry(image) - obj, found, err = p.cache.GetByKey(registry) + + cacheKey, err = generateCacheKey(registry, serviceAccountCacheKey) + if err != nil { + return nil, false, fmt.Errorf("error generating cache key: %w", err) + } + + obj, found, err = p.cache.GetByKey(cacheKey) if err != nil { return nil, false, err } @@ -340,7 +552,12 @@ func (p *pluginProvider) getCachedCredentials(image string) (credentialprovider. return obj.(*cacheEntry).credentials, true, nil } - obj, found, err = p.cache.GetByKey(globalCacheKey) + cacheKey, err = generateCacheKey(globalCacheKey, serviceAccountCacheKey) + if err != nil { + return nil, false, fmt.Errorf("error generating cache key: %w", err) + } + + obj, found, err = p.cache.GetByKey(cacheKey) if err != nil { return nil, false, err } @@ -355,7 +572,7 @@ func (p *pluginProvider) getCachedCredentials(image string) (credentialprovider. // Plugin is the interface calling ExecPlugin. This is mainly for testability // so tests don't have to actually exec any processes. type Plugin interface { - ExecPlugin(ctx context.Context, image string) (*credentialproviderapi.CredentialProviderResponse, error) + ExecPlugin(ctx context.Context, image, serviceAccountToken string, serviceAccountAnnotations map[string]string) (*credentialproviderapi.CredentialProviderResponse, error) } // execPlugin is the implementation of the Plugin interface that execs a credential provider plugin based @@ -377,10 +594,10 @@ type execPlugin struct { // // The plugin is expected to receive the CredentialProviderRequest API via stdin from the kubelet and // return CredentialProviderResponse via stdout. -func (e *execPlugin) ExecPlugin(ctx context.Context, image string) (*credentialproviderapi.CredentialProviderResponse, error) { +func (e *execPlugin) ExecPlugin(ctx context.Context, image, serviceAccountToken string, serviceAccountAnnotations map[string]string) (*credentialproviderapi.CredentialProviderResponse, error) { klog.V(5).Infof("Getting image %s credentials from external exec plugin %s", image, e.name) - authRequest := &credentialproviderapi.CredentialProviderRequest{Image: image} + authRequest := &credentialproviderapi.CredentialProviderRequest{Image: image, ServiceAccountToken: serviceAccountToken, ServiceAccountAnnotations: serviceAccountAnnotations} data, err := e.encodeRequest(authRequest) if err != nil { return nil, fmt.Errorf("failed to encode auth request: %w", err) @@ -499,3 +716,96 @@ func mergeEnvVars(sysEnvVars, credProviderVars []string) []string { mergedEnvVars = append(mergedEnvVars, credProviderVars...) return mergedEnvVars } + +// generateServiceAccountCacheKey generates the serviceaccount cache key to be used for +// 1. constructing the cache key for the service account token based plugin in addition to the actual cache key (image, registry, global). +// 2. the unique key to use singleflight for the plugin in addition to the image. +func generateServiceAccountCacheKey(serviceAccountNamespace, serviceAccountName string, serviceAccountUID types.UID, saAnnotations map[string]string) (string, error) { + b := cryptobyte.NewBuilder(nil) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(serviceAccountNamespace)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(serviceAccountName)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(serviceAccountUID)) + }) + + // add the length of annotations to the cache key + b.AddUint32(uint32(len(saAnnotations))) + + // Sort the annotations by key to ensure the cache key is deterministic + keys := sets.StringKeySet(saAnnotations).List() + for _, k := range keys { + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(k)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(saAnnotations[k])) + }) + } + + keyBytes, err := b.Bytes() + if err != nil { + return "", err + } + + return string(keyBytes), nil +} + +func generateCacheKey(baseKey, serviceAccountCacheKey string) (string, error) { + b := cryptobyte.NewBuilder(nil) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(baseKey)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(serviceAccountCacheKey)) + }) + + keyBytes, err := b.Bytes() + if err != nil { + return "", err + } + + return string(keyBytes), nil +} + +func generateSingleFlightKey(image, saTokenHash string, saAnnotations map[string]string) (string, error) { + b := cryptobyte.NewBuilder(nil) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(image)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(saTokenHash)) + }) + + // add the length of annotations to the cache key + b.AddUint32(uint32(len(saAnnotations))) + + // Sort the annotations by key to ensure the cache key is deterministic + keys := sets.StringKeySet(saAnnotations).List() + for _, k := range keys { + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(k)) + }) + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes([]byte(saAnnotations[k])) + }) + } + + keyBytes, err := b.Bytes() + if err != nil { + return "", err + } + + return string(keyBytes), nil +} + +// getHashIfNotEmpty returns the sha256 hash of the data if it is not empty. +func getHashIfNotEmpty(data string) string { + if len(data) > 0 { + return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data))) + } + return "" +} diff --git a/pkg/credentialprovider/plugin/plugin_test.go b/pkg/credentialprovider/plugin/plugin_test.go index 1f02992b11d..5a7857a0cbc 100644 --- a/pkg/credentialprovider/plugin/plugin_test.go +++ b/pkg/credentialprovider/plugin/plugin_test.go @@ -17,6 +17,7 @@ limitations under the License. package plugin import ( + "bytes" "context" "fmt" "reflect" @@ -24,9 +25,14 @@ import ( "testing" "time" + "golang.org/x/sync/singleflight" + + authenticationv1 "k8s.io/api/authentication/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/tools/cache" credentialproviderapi "k8s.io/kubelet/pkg/apis/credentialprovider" @@ -46,7 +52,16 @@ type fakeExecPlugin struct { auth map[string]credentialproviderapi.AuthConfig } -func (f *fakeExecPlugin) ExecPlugin(ctx context.Context, image string) (*credentialproviderapi.CredentialProviderResponse, error) { +// countingFakeExecPlugin is a fakeExecPlugin that counts the number of times ExecPlugin is called +// and sleeps for a second to simulate a slow plugin so that concurrent calls exercise the singleflight. +// This is used to test the singleflight behavior in the perPodPluginProvider. +type countingFakeExecPlugin struct { + fakeExecPlugin + mu sync.Mutex + count int +} + +func (f *fakeExecPlugin) ExecPlugin(ctx context.Context, image, serviceAccountToken string, serviceAccountAnnotations map[string]string) (*credentialproviderapi.CredentialProviderResponse, error) { return &credentialproviderapi.CredentialProviderResponse{ CacheKeyType: f.cacheKeyType, CacheDuration: &metav1.Duration{ @@ -56,27 +71,145 @@ func (f *fakeExecPlugin) ExecPlugin(ctx context.Context, image string) (*credent }, nil } +func (f *countingFakeExecPlugin) ExecPlugin(ctx context.Context, image, serviceAccountToken string, serviceAccountAnnotations map[string]string) (*credentialproviderapi.CredentialProviderResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.count++ + // make the exec plugin slow so concurrent calls exercise the singleflight + time.Sleep(time.Second) + return f.fakeExecPlugin.ExecPlugin(ctx, image, serviceAccountToken, serviceAccountAnnotations) +} + +func TestSingleflightProvide(t *testing.T) { + tclock := clock.RealClock{} + + // Set up the counting fakeExecPlugin + execPlugin := &countingFakeExecPlugin{ + fakeExecPlugin: fakeExecPlugin{ + cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, + auth: map[string]credentialproviderapi.AuthConfig{ + "test.registry.io": {Username: "user", Password: "password"}, + }, + }, + } + + // Set up perPodPluginProvider + pluginProvider := &pluginProvider{ + plugin: execPlugin, + group: singleflight.Group{}, + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"test.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + } + dynamicProvider := &perPodPluginProvider{ + provider: pluginProvider, + + podName: "pod-name", + podNamespace: "pod-namespace", + podUID: "pod-uid", + serviceAccountName: "service-account-name", + } + + image := "test.registry.io" + var wg sync.WaitGroup + const concurrentCalls = 5 + results := make([]credentialprovider.DockerConfig, concurrentCalls) + + // Test with serviceAccountProvider as nil + for i := 0; i < concurrentCalls; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + result := dynamicProvider.Provide(image) + results[i] = result + }(i) + } + wg.Wait() + + // Check that ExecPlugin was called only once + if execPlugin.count != 1 { + t.Errorf("expected ExecPlugin to be called once, but was called %d times", execPlugin.count) + } + + // Repeat the test with a non-nil serviceAccountProvider if applicable + pluginProvider.serviceAccountProvider = &serviceAccountProvider{ + audience: "audience", + getServiceAccountFunc: func(namespace, name string) (*v1.ServiceAccount, error) { + return &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: "service-account-uid", + }, + }, nil + }, + getServiceAccountTokenFunc: func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) { + return &authenticationv1.TokenRequest{}, nil + }, + } + + execPlugin.count = 0 // Reset count for the next test + for i := 0; i < concurrentCalls; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + result := dynamicProvider.Provide(image) + results[i] = result + }(i) + } + wg.Wait() + + // Verify single ExecPlugin call again + if execPlugin.count != 1 { + t.Errorf("expected ExecPlugin to be called once with serviceAccountProvider, but was called %d times", execPlugin.count) + } + + // Repeat the test with different serviceaccount token (same serviceaccount but different pod) + pluginProvider.serviceAccountProvider.getServiceAccountTokenFunc = func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) { + return &authenticationv1.TokenRequest{Status: authenticationv1.TokenRequestStatus{Token: rand.String(10)}}, nil + } + + execPlugin.count = 0 // Reset count for the next test + for i := 0; i < concurrentCalls; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + result := dynamicProvider.Provide(image) + results[i] = result + }(i) + } + wg.Wait() + + // Check that ExecPlugin was called 5 times with different serviceaccount tokens + if execPlugin.count != concurrentCalls { + t.Errorf("expected ExecPlugin to be called %d times with different serviceaccount tokens, but was called %d times", concurrentCalls, execPlugin.count) + } +} + func Test_Provide(t *testing.T) { tclock := clock.RealClock{} testcases := []struct { name string - pluginProvider *pluginProvider + pluginProvider *perPodPluginProvider image string dockerconfig credentialprovider.DockerConfig }{ { name: "exact image match, with Registry cache key", - pluginProvider: &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"test.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, - auth: map[string]credentialproviderapi.AuthConfig{ - "test.registry.io": { - Username: "user", - Password: "password", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"test.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, + auth: map[string]credentialproviderapi.AuthConfig{ + "test.registry.io": { + Username: "user", + Password: "password", + }, }, }, }, @@ -91,17 +224,19 @@ func Test_Provide(t *testing.T) { }, { name: "exact image match, with Image cache key", - pluginProvider: &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"test.registry.io/foo/bar"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.ImagePluginCacheKeyType, - auth: map[string]credentialproviderapi.AuthConfig{ - "test.registry.io/foo/bar": { - Username: "user", - Password: "password", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"test.registry.io/foo/bar"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.ImagePluginCacheKeyType, + auth: map[string]credentialproviderapi.AuthConfig{ + "test.registry.io/foo/bar": { + Username: "user", + Password: "password", + }, }, }, }, @@ -116,17 +251,19 @@ func Test_Provide(t *testing.T) { }, { name: "exact image match, with Global cache key", - pluginProvider: &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"test.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, - auth: map[string]credentialproviderapi.AuthConfig{ - "test.registry.io": { - Username: "user", - Password: "password", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"test.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, + auth: map[string]credentialproviderapi.AuthConfig{ + "test.registry.io": { + Username: "user", + Password: "password", + }, }, }, }, @@ -141,17 +278,19 @@ func Test_Provide(t *testing.T) { }, { name: "wild card image match, with Registry cache key", - pluginProvider: &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"*.registry.io:8080"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, - auth: map[string]credentialproviderapi.AuthConfig{ - "*.registry.io:8080": { - Username: "user", - Password: "password", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io:8080"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io:8080": { + Username: "user", + Password: "password", + }, }, }, }, @@ -166,17 +305,19 @@ func Test_Provide(t *testing.T) { }, { name: "wild card image match, with Image cache key", - pluginProvider: &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"*.*.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.ImagePluginCacheKeyType, - auth: map[string]credentialproviderapi.AuthConfig{ - "*.*.registry.io": { - Username: "user", - Password: "password", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.ImagePluginCacheKeyType, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.*.registry.io": { + Username: "user", + Password: "password", + }, }, }, }, @@ -191,17 +332,19 @@ func Test_Provide(t *testing.T) { }, { name: "wild card image match, with Global cache key", - pluginProvider: &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"*.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, - auth: map[string]credentialproviderapi.AuthConfig{ - "*.registry.io": { - Username: "user", - Password: "password", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io": { + Username: "user", + Password: "password", + }, }, }, }, @@ -257,18 +400,20 @@ func Test_ProvideParallel(t *testing.T) { }, } - pluginProvider := &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"test1.registry.io", "test2.registry.io", "test3.registry.io", "test4.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheDuration: time.Minute * 1, - cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, - auth: map[string]credentialproviderapi.AuthConfig{ - "test.registry.io": { - Username: "user", - Password: "password", + pluginProvider := &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"test1.registry.io", "test2.registry.io", "test3.registry.io", "test4.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheDuration: time.Minute * 1, + cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, + auth: map[string]credentialproviderapi.AuthConfig{ + "test.registry.io": { + Username: "user", + Password: "password", + }, }, }, }, @@ -335,7 +480,7 @@ func Test_getCachedCredentials(t *testing.T) { }, }, cacheEntry: cacheEntry{ - key: "image1", + key: "\x00\x06image1\x00\x00", expiresAt: fakeClock.Now().Add(1 * time.Minute), credentials: map[string]credentialprovider.DockerConfigEntry{ "image1": { @@ -352,7 +497,7 @@ func Test_getCachedCredentials(t *testing.T) { getKey: "image2", keyLength: 1, cacheEntry: cacheEntry{ - key: "image2", + key: "\x00\x06image2\x00\x00", expiresAt: fakeClock.Now(), credentials: map[string]credentialprovider.DockerConfigEntry{ "image2": { @@ -372,7 +517,7 @@ func Test_getCachedCredentials(t *testing.T) { // get only, we will not be able verify the purge call. getKey: "random", cacheEntry: cacheEntry{ - key: "image3", + key: "\x00\x06image3\x00\x00", expiresAt: fakeClock.Now().Add(2 * time.Minute), credentials: map[string]credentialprovider.DockerConfigEntry{ "image3": { @@ -390,7 +535,142 @@ func Test_getCachedCredentials(t *testing.T) { fakeClock.Step(tc.step) // getCachedCredentials returns unexpired credentials. - res, _, err := p.getCachedCredentials(tc.getKey) + res, _, err := p.getCachedCredentials(tc.getKey, "") + if err != nil { + t.Errorf("Unexpected error %v", err) + } + if !reflect.DeepEqual(res, tc.expectedResponse) { + t.Logf("response %v", res) + t.Logf("expected response %v", tc.expectedResponse) + t.Errorf("Unexpected response") + } + + // Listkeys returns all the keys present in cache including expired keys. + if len(p.cache.ListKeys()) != tc.keyLength { + t.Errorf("Unexpected cache key length") + } + }) + } +} + +func Test_getCachedCredentials_pluginUsingServiceAccount(t *testing.T) { + fakeClock := testingclock.NewFakeClock(time.Now()) + + p := &pluginProvider{ + clock: fakeClock, + lastCachePurge: fakeClock.Now(), + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: fakeClock}), + plugin: &fakeExecPlugin{}, + serviceAccountProvider: &serviceAccountProvider{ + audience: "audience", + getServiceAccountFunc: func(namespace, name string) (*v1.ServiceAccount, error) { + return &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: "service-account-uid", + }, + }, nil + }, + getServiceAccountTokenFunc: func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) { + return &authenticationv1.TokenRequest{}, nil + }, + }, + } + + serviceAccountCacheKey, err := generateServiceAccountCacheKey("namespace", "serviceAccountName", "service-account-uid", map[string]string{"prefix.io/annotation-1": "value1", "prefix.io/annotation-2": "value2"}) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + + cacheKey1, err := generateCacheKey("image1", serviceAccountCacheKey) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + + cacheKey2, err := generateCacheKey("image2", serviceAccountCacheKey) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + + testcases := []struct { + name string + step time.Duration + cacheEntry cacheEntry + expectedResponse credentialprovider.DockerConfig + keyLength int + getKey string + }{ + { + name: "It should return not expired credential", + step: 1 * time.Second, + keyLength: 1, + getKey: "image1", + expectedResponse: map[string]credentialprovider.DockerConfigEntry{ + "image1": { + Username: "user1", + Password: "pass1", + }, + }, + cacheEntry: cacheEntry{ + key: cacheKey1, + expiresAt: fakeClock.Now().Add(1 * time.Minute), + credentials: map[string]credentialprovider.DockerConfigEntry{ + "image1": { + Username: "user1", + Password: "pass1", + }, + }, + }, + }, + + { + name: "It should not return expired credential", + step: 2 * time.Minute, + getKey: "image2", + keyLength: 1, + cacheEntry: cacheEntry{ + key: cacheKey2, + expiresAt: fakeClock.Now(), + credentials: map[string]credentialprovider.DockerConfigEntry{ + "image2": { + Username: "user2", + Password: "pass2", + }, + }, + }, + }, + + { + name: "It should delete expired credential during purge", + step: 18 * time.Minute, + keyLength: 0, + // while get call for random, cache purge will be called, and it will delete expired + // image3 credentials. We cannot use image3 as getKey here, as it will get deleted during + // get only, we will not be able to verify the purge call. + getKey: "random", + cacheEntry: cacheEntry{ + key: "image3", + expiresAt: fakeClock.Now().Add(2 * time.Minute), + credentials: map[string]credentialprovider.DockerConfigEntry{ + "image3": { + Username: "user3", + Password: "pass3", + }, + }, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + if err := p.cache.Add(&tc.cacheEntry); err != nil { + t.Fatalf("Unexpected error %v", err) + } + fakeClock.Step(tc.step) + + // getCachedCredentials returns unexpired credentials. + res, _, err := p.getCachedCredentials(tc.getKey, serviceAccountCacheKey) if err != nil { t.Errorf("Unexpected error %v", err) } @@ -443,6 +723,20 @@ func Test_encodeRequest(t *testing.T) { Image: "test.registry.io/foobar", }, expectedData: []byte(`{"kind":"CredentialProviderRequest","apiVersion":"credentialprovider.kubelet.k8s.io/v1","image":"test.registry.io/foobar"} +`), + expectedErr: false, + }, + { + name: "successful with v1, with service account token and annotations", + apiVersion: credentialproviderv1.SchemeGroupVersion, + request: &credentialproviderapi.CredentialProviderRequest{ + Image: "test.registry.io/foobar", + ServiceAccountToken: "service-account-token", + ServiceAccountAnnotations: map[string]string{ + "domain.io/annotation1": "value1", + }, + }, + expectedData: []byte(`{"kind":"CredentialProviderRequest","apiVersion":"credentialprovider.kubelet.k8s.io/v1","image":"test.registry.io/foobar","serviceAccountToken":"service-account-token","serviceAccountAnnotations":{"domain.io/annotation1":"value1"}} `), expectedErr: false, }, @@ -574,177 +868,416 @@ func Test_decodeResponse(t *testing.T) { func Test_RegistryCacheKeyType(t *testing.T) { tclock := clock.RealClock{} - pluginProvider := &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"*.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, - cacheDuration: time.Hour, - auth: map[string]credentialproviderapi.AuthConfig{ - "*.registry.io": { - Username: "user", - Password: "password", + + tests := []struct { + name string + pluginProvider *perPodPluginProvider + expectedCacheKeys func(p *pluginProvider) []string + }{ + { + name: "plugin not using service account token", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, + cacheDuration: time.Hour, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io": { + Username: "user", + Password: "password", + }, + }, + }, }, + podName: "pod-name", + podNamespace: "namespace", + podUID: types.UID("pod-uid"), + serviceAccountName: "service-account-name", + }, + expectedCacheKeys: func(p *pluginProvider) []string { + return []string{"\x00\x10test.registry.io\x00\x00"} + }, + }, + { + name: "plugin using service account token", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + serviceAccountProvider: &serviceAccountProvider{ + audience: "audience", + requiredServiceAccountAnnotationKeys: []string{"prefix.io/annotation-1", "prefix.io/annotation-2"}, + getServiceAccountFunc: func(namespace, name string) (*v1.ServiceAccount, error) { + return &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: "service-account-uid", + Annotations: map[string]string{ + "prefix.io/annotation-1": "value1", + "prefix.io/annotation-2": "value2", + }, + }, + }, nil + }, + getServiceAccountTokenFunc: func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) { + return &authenticationv1.TokenRequest{}, nil + }, + }, + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.RegistryPluginCacheKeyType, + cacheDuration: time.Hour, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io": { + Username: "user", + Password: "password", + }, + }, + }, + }, + podName: "pod-name", + podNamespace: "namespace", + podUID: types.UID("pod-uid"), + serviceAccountName: "service-account-name", + }, + expectedCacheKeys: func(p *pluginProvider) []string { + serviceAccountCacheKey, err := generateServiceAccountCacheKey("namespace", "service-account-name", "service-account-uid", map[string]string{"prefix.io/annotation-1": "value1", "prefix.io/annotation-2": "value2"}) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + cacheKey, err := generateCacheKey("test.registry.io", serviceAccountCacheKey) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return []string{cacheKey} }, }, } - expectedDockerConfig := credentialprovider.DockerConfig{ - "*.registry.io": credentialprovider.DockerConfigEntry{ - Username: "user", - Password: "password", - }, - } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + expectedDockerConfig := credentialprovider.DockerConfig{ + "*.registry.io": credentialprovider.DockerConfigEntry{ + Username: "user", + Password: "password", + }, + } - dockerConfig := pluginProvider.Provide("test.registry.io/foo/bar") - if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { - t.Logf("actual docker config: %v", dockerConfig) - t.Logf("expected docker config: %v", expectedDockerConfig) - t.Fatal("unexpected docker config") - } + dockerConfig := test.pluginProvider.Provide("test.registry.io/foo/bar") + if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { + t.Logf("actual docker config: %v", dockerConfig) + t.Logf("expected docker config: %v", expectedDockerConfig) + t.Fatal("unexpected docker config") + } - expectedCacheKeys := []string{"test.registry.io"} - cacheKeys := pluginProvider.cache.ListKeys() + cacheKeys := test.pluginProvider.provider.cache.ListKeys() - if !reflect.DeepEqual(cacheKeys, expectedCacheKeys) { - t.Logf("actual cache keys: %v", cacheKeys) - t.Logf("expected cache keys: %v", expectedCacheKeys) - t.Error("unexpected cache keys") - } + expectedCacheKeys := test.expectedCacheKeys(test.pluginProvider.provider) + if !reflect.DeepEqual(cacheKeys, expectedCacheKeys) { + t.Logf("actual cache keys: %#v", cacheKeys) + t.Logf("expected cache keys: %v", expectedCacheKeys) + t.Error("unexpected cache keys") + } - // nil out the exec plugin, this will test whether credentialproviderapi are fetched - // from cache, otherwise Provider should panic - pluginProvider.plugin = nil - dockerConfig = pluginProvider.Provide("test.registry.io/foo/bar") - if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { - t.Logf("actual docker config: %v", dockerConfig) - t.Logf("expected docker config: %v", expectedDockerConfig) - t.Fatal("unexpected docker config") + // nil out the exec plugin, this will test whether credentialproviderapi are fetched + // from cache, otherwise Provider should panic + test.pluginProvider.provider.plugin = nil + dockerConfig = test.pluginProvider.Provide("test.registry.io/foo/bar") + if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { + t.Logf("actual docker config: %v", dockerConfig) + t.Logf("expected docker config: %v", expectedDockerConfig) + t.Fatal("unexpected docker config") + } + }) } } func Test_ImageCacheKeyType(t *testing.T) { tclock := clock.RealClock{} - pluginProvider := &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"*.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.ImagePluginCacheKeyType, - cacheDuration: time.Hour, - auth: map[string]credentialproviderapi.AuthConfig{ - "*.registry.io": { - Username: "user", - Password: "password", + + tests := []struct { + name string + pluginProvider *perPodPluginProvider + expectedCacheKeys func(p *pluginProvider) []string + }{ + { + name: "plugin not using service account token", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.ImagePluginCacheKeyType, + cacheDuration: time.Hour, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io": { + Username: "user", + Password: "password", + }, + }, + }, }, + podName: "pod-name", + podNamespace: "namespace", + podUID: types.UID("pod-uid"), + serviceAccountName: "service-account-name", + }, + expectedCacheKeys: func(p *pluginProvider) []string { + return []string{"\x00\x18test.registry.io/foo/bar\x00\x00"} + }, + }, + { + name: "plugin using service account token", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.ImagePluginCacheKeyType, + cacheDuration: time.Hour, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io": { + Username: "user", + Password: "password", + }, + }, + }, + serviceAccountProvider: &serviceAccountProvider{ + audience: "audience", + requiredServiceAccountAnnotationKeys: []string{"prefix.io/annotation-1", "prefix.io/annotation-2"}, + getServiceAccountFunc: func(namespace, name string) (*v1.ServiceAccount, error) { + return &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: "service-account-uid", + Annotations: map[string]string{ + "prefix.io/annotation-1": "value1", + "prefix.io/annotation-2": "value2", + }, + }, + }, nil + }, + getServiceAccountTokenFunc: func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) { + return &authenticationv1.TokenRequest{}, nil + }, + }, + }, + podName: "pod-name", + podNamespace: "namespace", + podUID: types.UID("pod-uid"), + serviceAccountName: "service-account-name", + }, + expectedCacheKeys: func(p *pluginProvider) []string { + serviceAccountCacheKey, err := generateServiceAccountCacheKey("namespace", "service-account-name", "service-account-uid", map[string]string{"prefix.io/annotation-1": "value1", "prefix.io/annotation-2": "value2"}) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + cacheKey, err := generateCacheKey("test.registry.io/foo/bar", serviceAccountCacheKey) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return []string{cacheKey} }, }, } - expectedDockerConfig := credentialprovider.DockerConfig{ - "*.registry.io": credentialprovider.DockerConfigEntry{ - Username: "user", - Password: "password", - }, - } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + expectedDockerConfig := credentialprovider.DockerConfig{ + "*.registry.io": credentialprovider.DockerConfigEntry{ + Username: "user", + Password: "password", + }, + } - dockerConfig := pluginProvider.Provide("test.registry.io/foo/bar") - if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { - t.Logf("actual docker config: %v", dockerConfig) - t.Logf("expected docker config: %v", expectedDockerConfig) - t.Fatal("unexpected docker config") - } + dockerConfig := test.pluginProvider.Provide("test.registry.io/foo/bar") + if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { + t.Logf("actual docker config: %v", dockerConfig) + t.Logf("expected docker config: %v", expectedDockerConfig) + t.Fatal("unexpected docker config") + } - expectedCacheKeys := []string{"test.registry.io/foo/bar"} - cacheKeys := pluginProvider.cache.ListKeys() + cacheKeys := test.pluginProvider.provider.cache.ListKeys() - if !reflect.DeepEqual(cacheKeys, expectedCacheKeys) { - t.Logf("actual cache keys: %v", cacheKeys) - t.Logf("expected cache keys: %v", expectedCacheKeys) - t.Error("unexpected cache keys") - } + expectedCacheKeys := test.expectedCacheKeys(test.pluginProvider.provider) + if !reflect.DeepEqual(cacheKeys, expectedCacheKeys) { + t.Logf("actual cache keys: %#v", cacheKeys) + t.Logf("expected cache keys: %v", expectedCacheKeys) + t.Error("unexpected cache keys") + } - // nil out the exec plugin, this will test whether credentialproviderapi are fetched - // from cache, otherwise Provider should panic - pluginProvider.plugin = nil - dockerConfig = pluginProvider.Provide("test.registry.io/foo/bar") - if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { - t.Logf("actual docker config: %v", dockerConfig) - t.Logf("expected docker config: %v", expectedDockerConfig) - t.Fatal("unexpected docker config") + // nil out the exec plugin, this will test whether credentialproviderapi are fetched + // from cache, otherwise Provider should panic + test.pluginProvider.provider.plugin = nil + dockerConfig = test.pluginProvider.Provide("test.registry.io/foo/bar") + if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { + t.Logf("actual docker config: %v", dockerConfig) + t.Logf("expected docker config: %v", expectedDockerConfig) + t.Fatal("unexpected docker config") + } + }) } } func Test_GlobalCacheKeyType(t *testing.T) { tclock := clock.RealClock{} - pluginProvider := &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"*.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, - cacheDuration: time.Hour, - auth: map[string]credentialproviderapi.AuthConfig{ - "*.registry.io": { - Username: "user", - Password: "password", + + tests := []struct { + name string + pluginProvider *perPodPluginProvider + expectedCacheKeys func(p *pluginProvider) []string + }{ + { + name: "plugin not using service account token", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, + cacheDuration: time.Hour, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io": { + Username: "user", + Password: "password", + }, + }, + }, }, + podName: "pod-name", + podNamespace: "namespace", + podUID: types.UID("pod-uid"), + serviceAccountName: "service-account-name", + }, + expectedCacheKeys: func(p *pluginProvider) []string { + return []string{"\x00\x06global\x00\x00"} + }, + }, + { + name: "plugin using service account token", + pluginProvider: &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, + cacheDuration: time.Hour, + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io": { + Username: "user", + Password: "password", + }, + }, + }, + serviceAccountProvider: &serviceAccountProvider{ + audience: "audience", + requiredServiceAccountAnnotationKeys: []string{"prefix.io/annotation-1", "prefix.io/annotation-2"}, + getServiceAccountFunc: func(namespace, name string) (*v1.ServiceAccount, error) { + return &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: "service-account-uid", + Annotations: map[string]string{ + "prefix.io/annotation-1": "value1", + "prefix.io/annotation-2": "value2", + }, + }, + }, nil + }, + getServiceAccountTokenFunc: func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) { + return &authenticationv1.TokenRequest{}, nil + }, + }, + }, + podName: "pod-name", + podNamespace: "namespace", + podUID: types.UID("pod-uid"), + serviceAccountName: "service-account-name", + }, + expectedCacheKeys: func(p *pluginProvider) []string { + serviceAccountCacheKey, err := generateServiceAccountCacheKey("namespace", "service-account-name", "service-account-uid", map[string]string{"prefix.io/annotation-1": "value1", "prefix.io/annotation-2": "value2"}) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + cacheKey, err := generateCacheKey(globalCacheKey, serviceAccountCacheKey) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return []string{cacheKey} }, }, } - expectedDockerConfig := credentialprovider.DockerConfig{ - "*.registry.io": credentialprovider.DockerConfigEntry{ - Username: "user", - Password: "password", - }, - } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + expectedDockerConfig := credentialprovider.DockerConfig{ + "*.registry.io": credentialprovider.DockerConfigEntry{ + Username: "user", + Password: "password", + }, + } - dockerConfig := pluginProvider.Provide("test.registry.io/foo/bar") - if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { - t.Logf("actual docker config: %v", dockerConfig) - t.Logf("expected docker config: %v", expectedDockerConfig) - t.Fatal("unexpected docker config") - } + dockerConfig := test.pluginProvider.Provide("test.registry.io/foo/bar") + if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { + t.Logf("actual docker config: %v", dockerConfig) + t.Logf("expected docker config: %v", expectedDockerConfig) + t.Fatal("unexpected docker config") + } - expectedCacheKeys := []string{"global"} - cacheKeys := pluginProvider.cache.ListKeys() + cacheKeys := test.pluginProvider.provider.cache.ListKeys() - if !reflect.DeepEqual(cacheKeys, expectedCacheKeys) { - t.Logf("actual cache keys: %v", cacheKeys) - t.Logf("expected cache keys: %v", expectedCacheKeys) - t.Error("unexpected cache keys") - } + expectedCacheKeys := test.expectedCacheKeys(test.pluginProvider.provider) + if !reflect.DeepEqual(cacheKeys, expectedCacheKeys) { + t.Logf("actual cache keys: %#v", cacheKeys) + t.Logf("expected cache keys: %v", expectedCacheKeys) + t.Error("unexpected cache keys") + } - // nil out the exec plugin, this will test whether credentialproviderapi are fetched - // from cache, otherwise Provider should panic - pluginProvider.plugin = nil - dockerConfig = pluginProvider.Provide("test.registry.io/foo/bar") - if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { - t.Logf("actual docker config: %v", dockerConfig) - t.Logf("expected docker config: %v", expectedDockerConfig) - t.Fatal("unexpected docker config") + // nil out the exec plugin, this will test whether credentialproviderapi are fetched + // from cache, otherwise Provider should panic + test.pluginProvider.provider.plugin = nil + dockerConfig = test.pluginProvider.Provide("test.registry.io/foo/bar") + if !reflect.DeepEqual(dockerConfig, expectedDockerConfig) { + t.Logf("actual docker config: %v", dockerConfig) + t.Logf("expected docker config: %v", expectedDockerConfig) + t.Fatal("unexpected docker config") + } + }) } } func Test_NoCacheResponse(t *testing.T) { tclock := clock.RealClock{} - pluginProvider := &pluginProvider{ - clock: tclock, - lastCachePurge: tclock.Now(), - matchImages: []string{"*.registry.io"}, - cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), - plugin: &fakeExecPlugin{ - cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, - cacheDuration: 0, // no cache - auth: map[string]credentialproviderapi.AuthConfig{ - "*.registry.io": { - Username: "user", - Password: "password", + pluginProvider := &perPodPluginProvider{ + provider: &pluginProvider{ + clock: tclock, + lastCachePurge: tclock.Now(), + matchImages: []string{"*.registry.io"}, + cache: cache.NewExpirationStore(cacheKeyFunc, &cacheExpirationPolicy{clock: tclock}), + plugin: &fakeExecPlugin{ + cacheKeyType: credentialproviderapi.GlobalPluginCacheKeyType, + cacheDuration: 0, // no cache + auth: map[string]credentialproviderapi.AuthConfig{ + "*.registry.io": { + Username: "user", + Password: "password", + }, }, }, }, @@ -765,7 +1298,7 @@ func Test_NoCacheResponse(t *testing.T) { } expectedCacheKeys := []string{} - cacheKeys := pluginProvider.cache.ListKeys() + cacheKeys := pluginProvider.provider.cache.ListKeys() if !reflect.DeepEqual(cacheKeys, expectedCacheKeys) { t.Logf("actual cache keys: %v", cacheKeys) t.Logf("expected cache keys: %v", expectedCacheKeys) @@ -879,3 +1412,159 @@ func validate(expected, actual []string) error { return nil } + +func TestGenerateServiceAccountCacheKey_Deterministic(t *testing.T) { + namespace1 := "default" + name1 := "service-account1" + uid1 := types.UID("633a81d0-0f58-4a43-9e84-113145201b72") + annotations1 := map[string]string{"domain.io/identity-id": "1234567890", "domain.io/role": "admin"} + + namespace2 := "kube-system" + name2 := "service-account2" + uid2 := types.UID("1408a4e6-e40b-4bbf-9019-4d86bfea73ae") + annotations2 := map[string]string{"domain.io/identity-id": "0987654321", "domain.io/role": "viewer"} + + testCases := []struct { + serviceAccountNamespace string + serviceAccountName string + serviceAccountUID types.UID + requiredAnnotations map[string]string + }{ + {namespace1, name1, uid1, annotations1}, + {namespace1, name1, uid1, annotations2}, + {namespace1, name2, uid1, annotations1}, + {namespace1, name2, uid1, annotations2}, + {namespace2, name1, uid1, annotations1}, + {namespace2, name1, uid1, annotations2}, + {namespace2, name2, uid1, annotations1}, + {namespace2, name2, uid1, annotations2}, + {namespace1, name1, uid2, annotations1}, + {namespace1, name1, uid2, annotations2}, + {namespace1, name2, uid2, annotations1}, + {namespace1, name2, uid2, annotations2}, + {namespace2, name1, uid2, annotations1}, + {namespace2, name1, uid2, annotations2}, + {namespace2, name2, uid2, annotations1}, + {namespace2, name2, uid2, annotations2}, + } + + for _, tc := range testCases { + tc := tc + for _, tc2 := range testCases { + tc2 := tc2 + t.Run(fmt.Sprintf("%+v-%+v", tc, tc2), func(t *testing.T) { + serviceAccountCacheKey1, err1 := generateServiceAccountCacheKey(tc.serviceAccountNamespace, tc.serviceAccountName, tc.serviceAccountUID, tc.requiredAnnotations) + serviceAccountCacheKey2, err2 := generateServiceAccountCacheKey(tc2.serviceAccountNamespace, tc2.serviceAccountName, tc2.serviceAccountUID, tc2.requiredAnnotations) + + if err1 != nil || err2 != nil { + t.Errorf("expected no error, but got err1=%v, err2=%v", err1, err2) + } + + if bytes.Equal([]byte(serviceAccountCacheKey1), []byte(serviceAccountCacheKey2)) != reflect.DeepEqual(tc, tc2) { + t.Errorf("expected %v, got %v", reflect.DeepEqual(tc, tc2), bytes.Equal([]byte(serviceAccountCacheKey1), []byte(serviceAccountCacheKey2))) + } + + cacheKey1, err1 := generateCacheKey("registry.io/image", serviceAccountCacheKey1) + cacheKey2, err2 := generateCacheKey("registry.io/image", serviceAccountCacheKey2) + + if err1 != nil || err2 != nil { + t.Errorf("expected no error, but got err1=%v, err2=%v", err1, err2) + } + + if bytes.Equal([]byte(cacheKey1), []byte(cacheKey2)) != reflect.DeepEqual(tc, tc2) { + t.Errorf("expected %v, got %v", reflect.DeepEqual(tc, tc2), bytes.Equal([]byte(cacheKey1), []byte(cacheKey2))) + } + }) + } + } +} + +func TestGenerateServiceAccountCacheKey(t *testing.T) { + tests := []struct { + name string + saNamespace string + saName string + saUID types.UID + saAnnotations map[string]string + want string + }{ + { + name: "no annotations", + saNamespace: "namespace", + saName: "service-account", + saUID: "service-account-uid", + saAnnotations: nil, + want: "\x00\tnamespace\x00\x0fservice-account\x00\x13service-account-uid\x00\x00\x00\x00", + }, + { + name: "single annotation", + saNamespace: "namespace", + saName: "service-account", + saUID: "service-account-uid", + saAnnotations: map[string]string{"domain.io/annotation-1": "value1"}, + want: "\x00\tnamespace\x00\x0fservice-account\x00\x13service-account-uid\x00\x00\x00\x01\x00\x16domain.io/annotation-1\x00\x06value1", + }, + { + name: "multiple annotations", + saNamespace: "namespace", + saName: "service-account", + saUID: "service-account-uid", + saAnnotations: map[string]string{"domain.io/annotation-1": "value1", "domain.io/annotation-2": "value2"}, + want: "\x00\tnamespace\x00\x0fservice-account\x00\x13service-account-uid\x00\x00\x00\x02\x00\x16domain.io/annotation-1\x00\x06value1\x00\x16domain.io/annotation-2\x00\x06value2", + }, + { + name: "annotations with different order should be sorted", + saNamespace: "namespace", + saName: "service-account", + saUID: "service-account-uid", + saAnnotations: map[string]string{"domain.io/annotation-2": "value2", "domain.io/annotation-1": "value1"}, + want: "\x00\tnamespace\x00\x0fservice-account\x00\x13service-account-uid\x00\x00\x00\x02\x00\x16domain.io/annotation-1\x00\x06value1\x00\x16domain.io/annotation-2\x00\x06value2", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := generateServiceAccountCacheKey(tc.saNamespace, tc.saName, tc.saUID, tc.saAnnotations) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != tc.want { + t.Errorf("expected %q, got '%q'", tc.want, got) + } + }) + } +} + +func TestGenerateCacheKey(t *testing.T) { + tests := []struct { + name string + baseKey string + serviceAccountCacheKey string + want string + }{ + { + name: "empty service account cache key", + baseKey: "registry.io/image", + serviceAccountCacheKey: "", + want: "\x00\x11registry.io/image\x00\x00", + }, + { + name: "combined key", + baseKey: "registry.io/image", + serviceAccountCacheKey: "\x00\tnamespace\x00\x0fservice-account\x00\x13service-account-uid\x00\x00\x00\x02\x00\x16domain.io/annotation-1\x00\x06value1\x00\x16domain.io/annotation-2\x00\x06value2", + want: "\x00\x11registry.io/image\x00u\x00\tnamespace\x00\x0fservice-account\x00\x13service-account-uid\x00\x00\x00\x02\x00\x16domain.io/annotation-1\x00\x06value1\x00\x16domain.io/annotation-2\x00\x06value2", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := generateCacheKey(tc.baseKey, tc.serviceAccountCacheKey) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if got != tc.want { + t.Errorf("expected %q, got %q", tc.want, got) + } + }) + } +} diff --git a/pkg/credentialprovider/plugin/plugins.go b/pkg/credentialprovider/plugin/plugins.go new file mode 100644 index 00000000000..0260158ae47 --- /dev/null +++ b/pkg/credentialprovider/plugin/plugins.go @@ -0,0 +1,98 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "sync" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/credentialprovider" + "k8s.io/kubernetes/pkg/features" +) + +type provider struct { + name string + impl *pluginProvider +} + +var providersMutex sync.RWMutex +var providers = make([]provider, 0) +var seenProviderNames = sets.NewString() + +func registerCredentialProviderPlugin(name string, p *pluginProvider) { + providersMutex.Lock() + defer providersMutex.Unlock() + + if seenProviderNames.Has(name) { + klog.Fatalf("Credential provider %q was registered twice", name) + } + seenProviderNames.Insert(name) + + providers = append(providers, provider{name, p}) + klog.V(4).Infof("Registered credential provider %q", name) +} + +type externalCredentialProviderKeyring struct { + providers []credentialprovider.DockerConfigProvider +} + +func NewExternalCredentialProviderDockerKeyring(podNamespace, podName, podUID, serviceAccountName string) credentialprovider.DockerKeyring { + providersMutex.RLock() + defer providersMutex.RUnlock() + + keyring := &externalCredentialProviderKeyring{ + providers: make([]credentialprovider.DockerConfigProvider, 0, len(providers)), + } + + for _, p := range providers { + if !p.impl.Enabled() { + continue + } + + pp := &perPodPluginProvider{ + name: p.name, + provider: p.impl, + } + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletServiceAccountTokenForCredentialProviders) { + klog.V(4).InfoS("Generating per pod credential provider", "provider", p.name, "podName", podName, "podNamespace", podNamespace, "podUID", podUID, "serviceAccountName", serviceAccountName) + + pp.podNamespace = podNamespace + pp.podName = podName + pp.podUID = types.UID(podUID) + pp.serviceAccountName = serviceAccountName + } else { + klog.V(4).InfoS("Generating credential provider", "provider", p.name) + } + + keyring.providers = append(keyring.providers, pp) + } + + return keyring +} + +func (k *externalCredentialProviderKeyring) Lookup(image string) ([]credentialprovider.AuthConfig, bool) { + keyring := &credentialprovider.BasicDockerKeyring{} + + for _, p := range k.providers { + keyring.Add(p.Provide(image)) + } + + return keyring.Lookup(image) +} diff --git a/pkg/credentialprovider/plugins.go b/pkg/credentialprovider/plugins.go deleted file mode 100644 index 05fb34d4c49..00000000000 --- a/pkg/credentialprovider/plugins.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package credentialprovider - -import ( - "sync" - - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" -) - -type provider struct { - name string - impl DockerConfigProvider -} - -// All registered credential providers. -var providersMutex sync.Mutex -var providers = make([]provider, 0) -var seenProviderNames = sets.NewString() - -// RegisterCredentialProvider is called by provider implementations on -// initialization to register themselves, like so: -// -// func init() { -// RegisterCredentialProvider("name", &myProvider{...}) -// } -func RegisterCredentialProvider(name string, p DockerConfigProvider) { - providersMutex.Lock() - defer providersMutex.Unlock() - - if seenProviderNames.Has(name) { - klog.Fatalf("Credential provider %q was registered twice", name) - } - seenProviderNames.Insert(name) - - providers = append(providers, provider{name, p}) - klog.V(4).Infof("Registered credential provider %q", name) -} - -// NewDockerKeyring creates a DockerKeyring to use for resolving credentials, -// which draws from the set of registered credential providers. -func NewDockerKeyring() DockerKeyring { - keyring := &providersDockerKeyring{ - Providers: make([]DockerConfigProvider, 0), - } - - for _, p := range providers { - if p.impl.Enabled() { - klog.V(4).Infof("Registering credential provider: %v", p.name) - keyring.Providers = append(keyring.Providers, p.impl) - } - } - - return keyring -} diff --git a/pkg/credentialprovider/provider.go b/pkg/credentialprovider/provider.go index 8c9ad347b72..636f942420c 100644 --- a/pkg/credentialprovider/provider.go +++ b/pkg/credentialprovider/provider.go @@ -42,15 +42,6 @@ type DockerConfigProvider interface { // A DockerConfigProvider that simply reads the .dockercfg file type defaultDockerConfigProvider struct{} -// init registers our default provider, which simply reads the .dockercfg file. -func init() { - RegisterCredentialProvider(".dockercfg", - &CachingDockerConfigProvider{ - Provider: &defaultDockerConfigProvider{}, - Lifetime: 5 * time.Minute, - }) -} - // CachingDockerConfigProvider implements DockerConfigProvider by composing // with another DockerConfigProvider and caching the DockerConfig it provides // for a pre-specified lifetime. @@ -107,3 +98,16 @@ func (d *CachingDockerConfigProvider) Provide(image string) DockerConfig { } return config } + +// NewDefaultDockerKeyring creates a DockerKeyring to use for resolving credentials, +// which returns the default credentials from the .dockercfg file. +func NewDefaultDockerKeyring() DockerKeyring { + return &providersDockerKeyring{ + Providers: []DockerConfigProvider{ + &CachingDockerConfigProvider{ + Provider: &defaultDockerConfigProvider{}, + Lifetime: 5 * time.Minute, + }, + }, + } +} diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index 3b14039f461..8daf904f47f 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -152,7 +152,7 @@ type StreamingRuntime interface { type ImageService interface { // PullImage pulls an image from the network to local storage using the supplied // secrets if necessary. It returns a reference (digest or ID) to the pulled image. - PullImage(ctx context.Context, image ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error) + PullImage(ctx context.Context, image ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) (string, error) // GetImageRef gets the reference (digest or ID) of the image which has already been in // the local storage. It returns ("", nil) if the image isn't in the local storage. GetImageRef(ctx context.Context, image ImageSpec) (string, error) diff --git a/pkg/kubelet/container/testing/fake_runtime.go b/pkg/kubelet/container/testing/fake_runtime.go index 52fbd709d99..f7889f06903 100644 --- a/pkg/kubelet/container/testing/fake_runtime.go +++ b/pkg/kubelet/container/testing/fake_runtime.go @@ -308,7 +308,7 @@ func (f *FakeRuntime) GetContainerLogs(_ context.Context, pod *v1.Pod, container return f.Err } -func (f *FakeRuntime) PullImage(ctx context.Context, image kubecontainer.ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error) { +func (f *FakeRuntime) PullImage(ctx context.Context, image kubecontainer.ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) (string, error) { f.Lock() f.CalledFunctions = append(f.CalledFunctions, "PullImage") if f.Err == nil { diff --git a/pkg/kubelet/container/testing/runtime_mock.go b/pkg/kubelet/container/testing/runtime_mock.go index 63bd6b4e790..8f91580eee5 100644 --- a/pkg/kubelet/container/testing/runtime_mock.go +++ b/pkg/kubelet/container/testing/runtime_mock.go @@ -990,9 +990,9 @@ func (_c *MockRuntime_ListPodSandboxMetrics_Call) RunAndReturn(run func(context. return _c } -// PullImage provides a mock function with given fields: ctx, image, pullSecrets, podSandboxConfig -func (_m *MockRuntime) PullImage(ctx context.Context, image container.ImageSpec, pullSecrets []corev1.Secret, podSandboxConfig *v1.PodSandboxConfig) (string, error) { - ret := _m.Called(ctx, image, pullSecrets, podSandboxConfig) +// PullImage provides a mock function with given fields: ctx, image, pullSecrets, podSandboxConfig, serviceAccountName +func (_m *MockRuntime) PullImage(ctx context.Context, image container.ImageSpec, pullSecrets []corev1.Secret, podSandboxConfig *v1.PodSandboxConfig, serviceAccountName string) (string, error) { + ret := _m.Called(ctx, image, pullSecrets, podSandboxConfig, serviceAccountName) if len(ret) == 0 { panic("no return value specified for PullImage") @@ -1000,17 +1000,17 @@ func (_m *MockRuntime) PullImage(ctx context.Context, image container.ImageSpec, var r0 string var r1 error - if rf, ok := ret.Get(0).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig) (string, error)); ok { - return rf(ctx, image, pullSecrets, podSandboxConfig) + if rf, ok := ret.Get(0).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig, string) (string, error)); ok { + return rf(ctx, image, pullSecrets, podSandboxConfig, serviceAccountName) } - if rf, ok := ret.Get(0).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig) string); ok { - r0 = rf(ctx, image, pullSecrets, podSandboxConfig) + if rf, ok := ret.Get(0).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig, string) string); ok { + r0 = rf(ctx, image, pullSecrets, podSandboxConfig, serviceAccountName) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig) error); ok { - r1 = rf(ctx, image, pullSecrets, podSandboxConfig) + if rf, ok := ret.Get(1).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig, string) error); ok { + r1 = rf(ctx, image, pullSecrets, podSandboxConfig, serviceAccountName) } else { r1 = ret.Error(1) } @@ -1028,13 +1028,14 @@ type MockRuntime_PullImage_Call struct { // - image container.ImageSpec // - pullSecrets []corev1.Secret // - podSandboxConfig *v1.PodSandboxConfig -func (_e *MockRuntime_Expecter) PullImage(ctx interface{}, image interface{}, pullSecrets interface{}, podSandboxConfig interface{}) *MockRuntime_PullImage_Call { - return &MockRuntime_PullImage_Call{Call: _e.mock.On("PullImage", ctx, image, pullSecrets, podSandboxConfig)} +// - serviceAccountName string +func (_e *MockRuntime_Expecter) PullImage(ctx interface{}, image interface{}, pullSecrets interface{}, podSandboxConfig interface{}, serviceAccountName interface{}) *MockRuntime_PullImage_Call { + return &MockRuntime_PullImage_Call{Call: _e.mock.On("PullImage", ctx, image, pullSecrets, podSandboxConfig, serviceAccountName)} } -func (_c *MockRuntime_PullImage_Call) Run(run func(ctx context.Context, image container.ImageSpec, pullSecrets []corev1.Secret, podSandboxConfig *v1.PodSandboxConfig)) *MockRuntime_PullImage_Call { +func (_c *MockRuntime_PullImage_Call) Run(run func(ctx context.Context, image container.ImageSpec, pullSecrets []corev1.Secret, podSandboxConfig *v1.PodSandboxConfig, serviceAccountName string)) *MockRuntime_PullImage_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(container.ImageSpec), args[2].([]corev1.Secret), args[3].(*v1.PodSandboxConfig)) + run(args[0].(context.Context), args[1].(container.ImageSpec), args[2].([]corev1.Secret), args[3].(*v1.PodSandboxConfig), args[4].(string)) }) return _c } @@ -1044,7 +1045,7 @@ func (_c *MockRuntime_PullImage_Call) Return(_a0 string, _a1 error) *MockRuntime return _c } -func (_c *MockRuntime_PullImage_Call) RunAndReturn(run func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig) (string, error)) *MockRuntime_PullImage_Call { +func (_c *MockRuntime_PullImage_Call) RunAndReturn(run func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig, string) (string, error)) *MockRuntime_PullImage_Call { _c.Call.Return(run) return _c } diff --git a/pkg/kubelet/images/helpers.go b/pkg/kubelet/images/helpers.go index b8005d14f17..f2524c67a73 100644 --- a/pkg/kubelet/images/helpers.go +++ b/pkg/kubelet/images/helpers.go @@ -44,9 +44,9 @@ type throttledImageService struct { limiter flowcontrol.RateLimiter } -func (ts throttledImageService) PullImage(ctx context.Context, image kubecontainer.ImageSpec, secrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error) { +func (ts throttledImageService) PullImage(ctx context.Context, image kubecontainer.ImageSpec, secrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) (string, error) { if ts.limiter.TryAccept() { - return ts.ImageService.PullImage(ctx, image, secrets, podSandboxConfig) + return ts.ImageService.PullImage(ctx, image, secrets, podSandboxConfig, serviceAccountName) } return "", fmt.Errorf("pull QPS exceeded") } diff --git a/pkg/kubelet/images/image_manager.go b/pkg/kubelet/images/image_manager.go index 684eba80356..c1576f434c3 100644 --- a/pkg/kubelet/images/image_manager.go +++ b/pkg/kubelet/images/image_manager.go @@ -175,7 +175,7 @@ func (m *imageManager) EnsureImageExists(ctx context.Context, objRef *v1.ObjectR m.logIt(objRef, v1.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("Pulling image %q", imgRef), klog.Info) startTime := time.Now() pullChan := make(chan pullResult) - m.puller.pullImage(ctx, spec, pullSecrets, pullChan, podSandboxConfig) + m.puller.pullImage(ctx, spec, pullSecrets, pullChan, podSandboxConfig, pod.Spec.ServiceAccountName) imagePullResult := <-pullChan if imagePullResult.err != nil { m.logIt(objRef, v1.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", imgRef, imagePullResult.err), klog.Warning) diff --git a/pkg/kubelet/images/puller.go b/pkg/kubelet/images/puller.go index 0c9bff5dc4e..26432669ced 100644 --- a/pkg/kubelet/images/puller.go +++ b/pkg/kubelet/images/puller.go @@ -34,7 +34,7 @@ type pullResult struct { } type imagePuller interface { - pullImage(context.Context, kubecontainer.ImageSpec, []v1.Secret, chan<- pullResult, *runtimeapi.PodSandboxConfig) + pullImage(context.Context, kubecontainer.ImageSpec, []v1.Secret, chan<- pullResult, *runtimeapi.PodSandboxConfig, string) } var _, _ imagePuller = ¶llelImagePuller{}, &serialImagePuller{} @@ -51,14 +51,14 @@ func newParallelImagePuller(imageService kubecontainer.ImageService, maxParallel return ¶llelImagePuller{imageService, make(chan struct{}, *maxParallelImagePulls)} } -func (pip *parallelImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig) { +func (pip *parallelImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) { go func() { if pip.tokens != nil { pip.tokens <- struct{}{} defer func() { <-pip.tokens }() } startTime := time.Now() - imageRef, err := pip.imageService.PullImage(ctx, spec, pullSecrets, podSandboxConfig) + imageRef, err := pip.imageService.PullImage(ctx, spec, pullSecrets, podSandboxConfig, serviceAccountName) var size uint64 if err == nil && imageRef != "" { // Getting the image size with best effort, ignoring the error. @@ -88,27 +88,29 @@ func newSerialImagePuller(imageService kubecontainer.ImageService) imagePuller { } type imagePullRequest struct { - ctx context.Context - spec kubecontainer.ImageSpec - pullSecrets []v1.Secret - pullChan chan<- pullResult - podSandboxConfig *runtimeapi.PodSandboxConfig + ctx context.Context + spec kubecontainer.ImageSpec + pullSecrets []v1.Secret + pullChan chan<- pullResult + podSandboxConfig *runtimeapi.PodSandboxConfig + serviceAccountName string } -func (sip *serialImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig) { +func (sip *serialImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) { sip.pullRequests <- &imagePullRequest{ - ctx: ctx, - spec: spec, - pullSecrets: pullSecrets, - pullChan: pullChan, - podSandboxConfig: podSandboxConfig, + ctx: ctx, + spec: spec, + pullSecrets: pullSecrets, + pullChan: pullChan, + podSandboxConfig: podSandboxConfig, + serviceAccountName: serviceAccountName, } } func (sip *serialImagePuller) processImagePullRequests() { for pullRequest := range sip.pullRequests { startTime := time.Now() - imageRef, err := sip.imageService.PullImage(pullRequest.ctx, pullRequest.spec, pullRequest.pullSecrets, pullRequest.podSandboxConfig) + imageRef, err := sip.imageService.PullImage(pullRequest.ctx, pullRequest.spec, pullRequest.pullSecrets, pullRequest.podSandboxConfig, pullRequest.serviceAccountName) var size uint64 if err == nil && imageRef != "" { // Getting the image size with best effort, ignoring the error. diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 88e3b310586..4a182c9a8ec 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -41,6 +41,7 @@ import ( "go.opentelemetry.io/otel/codes" semconv "go.opentelemetry.io/otel/semconv/v1.12.0" "go.opentelemetry.io/otel/trace" + "k8s.io/client-go/informers" "k8s.io/mount-utils" @@ -713,6 +714,19 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } } + tokenManager := token.NewManager(kubeDeps.KubeClient) + getServiceAccount := func(namespace, name string) (*v1.ServiceAccount, error) { + return nil, fmt.Errorf("get service account is not implemented") + } + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletServiceAccountTokenForCredentialProviders) { + getServiceAccount = func(namespace, name string) (*v1.ServiceAccount, error) { + if klet.kubeClient == nil { + return nil, errors.New("cannot get ServiceAccounts when kubelet is in standalone mode") + } + return klet.kubeClient.CoreV1().ServiceAccounts(namespace).Get(ctx, name, metav1.GetOptions{}) + } + } + runtime, err := kuberuntime.NewKubeGenericRuntimeManager( kubecontainer.FilterEventRecorder(kubeDeps.Recorder), klet.livenessManager, @@ -747,6 +761,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, *kubeCfg.MemoryThrottlingFactor, kubeDeps.PodStartupLatencyTracker, kubeDeps.TracerProvider, + tokenManager, + getServiceAccount, ) if err != nil { return nil, err @@ -876,8 +892,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps.Recorder) } - tokenManager := token.NewManager(kubeDeps.KubeClient) - var clusterTrustBundleManager clustertrustbundle.Manager = &clustertrustbundle.NoopManager{} if kubeDeps.KubeClient != nil && utilfeature.DefaultFeatureGate.Enabled(features.ClusterTrustBundleProjection) { clusterTrustBundleManager = clustertrustbundle.NewLazyInformerManager(ctx, kubeDeps.KubeClient, 2*int(kubeCfg.MaxPods)) diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index da00c6b3e9d..d8c59dd3a6d 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -40,6 +40,7 @@ import ( cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + core "k8s.io/client-go/testing" "k8s.io/mount-utils" @@ -3424,6 +3425,8 @@ func TestSyncPodSpans(t *testing.T) { *kubeCfg.MemoryThrottlingFactor, kubeletutil.NewPodStartupLatencyTracker(), tp, + token.NewManager(kubelet.kubeClient), + func(string, string) (*v1.ServiceAccount, error) { return nil, nil }, ) assert.NoError(t, err) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_image.go b/pkg/kubelet/kuberuntime/kuberuntime_image.go index 1a15ef0ba8c..22c49d12c11 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_image.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_image.go @@ -24,6 +24,8 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/credentialprovider" + credentialproviderplugin "k8s.io/kubernetes/pkg/credentialprovider/plugin" credentialprovidersecrets "k8s.io/kubernetes/pkg/credentialprovider/secrets" "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -32,14 +34,26 @@ import ( // PullImage pulls an image from the network to local storage using the supplied // secrets if necessary. -func (m *kubeGenericRuntimeManager) PullImage(ctx context.Context, image kubecontainer.ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error) { +func (m *kubeGenericRuntimeManager) PullImage(ctx context.Context, image kubecontainer.ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) (string, error) { img := image.Image repoToPull, _, _, err := parsers.ParseImageName(img) if err != nil { return "", err } - keyring, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, m.keyring) + // construct the dynamic keyring using the providers we have in the kubelet + var podName, podNamespace, podUID string + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletServiceAccountTokenForCredentialProviders) { + sandboxMetadata := podSandboxConfig.GetMetadata() + + podName = sandboxMetadata.Name + podNamespace = sandboxMetadata.Namespace + podUID = sandboxMetadata.Uid + } + + externalCredentialProviderKeyring := credentialproviderplugin.NewExternalCredentialProviderDockerKeyring(podNamespace, podName, podUID, serviceAccountName) + + keyring, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, credentialprovider.UnionDockerKeyring{m.keyring, externalCredentialProviderKeyring}) if err != nil { return "", err } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_image_test.go b/pkg/kubelet/kuberuntime/kuberuntime_image_test.go index 228f5cd8e4a..3dd38b6b7d4 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_image_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_image_test.go @@ -37,7 +37,7 @@ func TestPullImage(t *testing.T) { _, _, fakeManager, err := createTestRuntimeManager() assert.NoError(t, err) - imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil) + imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil, "") assert.NoError(t, err) assert.Equal(t, "busybox", imageRef) @@ -53,12 +53,12 @@ func TestPullImageWithError(t *testing.T) { assert.NoError(t, err) // trying to pull an image with an invalid name should return an error - imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: ":invalid"}, nil, nil) + imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: ":invalid"}, nil, nil, "") assert.Error(t, err) assert.Equal(t, "", imageRef) fakeImageService.InjectError("PullImage", fmt.Errorf("test-error")) - imageRef, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil) + imageRef, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil, "") assert.Error(t, err) assert.Equal(t, "", imageRef) @@ -75,7 +75,7 @@ func TestPullImageWithInvalidImageName(t *testing.T) { fakeImageService.SetFakeImages(imageList) for _, val := range imageList { ctx := context.Background() - imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: val}, nil, nil) + imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: val}, nil, nil, "") assert.Error(t, err) assert.Equal(t, "", imageRef) @@ -196,7 +196,7 @@ func TestRemoveImage(t *testing.T) { _, fakeImageService, fakeManager, err := createTestRuntimeManager() assert.NoError(t, err) - _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil) + _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil, "") assert.NoError(t, err) assert.Len(t, fakeImageService.Images, 1) @@ -219,7 +219,7 @@ func TestRemoveImageWithError(t *testing.T) { _, fakeImageService, fakeManager, err := createTestRuntimeManager() assert.NoError(t, err) - _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil) + _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil, "") assert.NoError(t, err) assert.Len(t, fakeImageService.Images, 1) @@ -324,7 +324,7 @@ func TestPullWithSecrets(t *testing.T) { _, fakeImageService, fakeManager, err := customTestRuntimeManager(builtInKeyRing) require.NoError(t, err) - _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: test.imageName}, test.passedSecrets, nil) + _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: test.imageName}, test.passedSecrets, nil, "") require.NoError(t, err) fakeImageService.AssertImagePulledWithAuth(t, &runtimeapi.ImageSpec{Image: test.imageName, Annotations: make(map[string]string)}, test.expectedAuth, description) } @@ -375,7 +375,7 @@ func TestPullWithSecretsWithError(t *testing.T) { fakeImageService.InjectError("PullImage", fmt.Errorf("test-error")) } - imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: test.imageName}, test.passedSecrets, nil) + imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: test.imageName}, test.passedSecrets, nil, "") assert.Error(t, err) assert.Equal(t, "", imageRef) @@ -398,7 +398,7 @@ func TestPullThenListWithAnnotations(t *testing.T) { }, } - _, err = fakeManager.PullImage(ctx, imageSpec, nil, nil) + _, err = fakeManager.PullImage(ctx, imageSpec, nil, nil, "") assert.NoError(t, err) images, err := fakeManager.ListImages(ctx) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 167fd4a8f80..55f6c086094 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -28,8 +28,6 @@ import ( cadvisorapi "github.com/google/cadvisor/info/v1" "go.opentelemetry.io/otel/trace" grpcstatus "google.golang.org/grpc/status" - crierror "k8s.io/cri-api/pkg/errors" - "k8s.io/klog/v2" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -44,7 +42,8 @@ import ( "k8s.io/component-base/logs/logreduction" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" - + crierror "k8s.io/cri-api/pkg/errors" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/api/legacyscheme" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/credentialprovider" @@ -62,6 +61,7 @@ import ( proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results" "k8s.io/kubernetes/pkg/kubelet/runtimeclass" "k8s.io/kubernetes/pkg/kubelet/sysctl" + "k8s.io/kubernetes/pkg/kubelet/token" "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/cache" "k8s.io/kubernetes/pkg/kubelet/util/format" @@ -223,6 +223,8 @@ func NewKubeGenericRuntimeManager( memoryThrottlingFactor float64, podPullingTimeRecorder images.ImagePodPullingTimeRecorder, tracerProvider trace.TracerProvider, + tokenManager *token.Manager, + getServiceAccount func(string, string) (*v1.ServiceAccount, error), ) (KubeGenericRuntime, error) { ctx := context.Background() runtimeService = newInstrumentedRuntimeService(runtimeService) @@ -277,12 +279,12 @@ func NewKubeGenericRuntimeManager( "apiVersion", typedVersion.RuntimeApiVersion) if imageCredentialProviderConfigFile != "" || imageCredentialProviderBinDir != "" { - if err := plugin.RegisterCredentialProviderPlugins(imageCredentialProviderConfigFile, imageCredentialProviderBinDir); err != nil { + if err := plugin.RegisterCredentialProviderPlugins(imageCredentialProviderConfigFile, imageCredentialProviderBinDir, tokenManager.GetServiceAccountToken, getServiceAccount); err != nil { klog.ErrorS(err, "Failed to register CRI auth plugins") os.Exit(1) } } - kubeRuntimeManager.keyring = credentialprovider.NewDockerKeyring() + kubeRuntimeManager.keyring = credentialprovider.NewDefaultDockerKeyring() kubeRuntimeManager.imagePuller = images.NewImageManager( kubecontainer.FilterEventRecorder(recorder), diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index c36a0e78f68..1d44da8b3f4 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -27,14 +27,13 @@ import ( "testing" "time" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - cadvisorapi "github.com/google/cadvisor/info/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" noopoteltrace "go.opentelemetry.io/otel/trace/noop" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource"