From a947c32783e9027922762d6f76f5c27926f9bc0a Mon Sep 17 00:00:00 2001 From: Owen Strain Date: Thu, 27 May 2021 23:35:51 +0000 Subject: [PATCH] Add feature gate to disable in-tree credential providers --- pkg/credentialprovider/aws/aws_credentials.go | 8 +++++++ .../azure/azure_credentials.go | 10 ++++++++ pkg/credentialprovider/gcp/metadata.go | 23 ++++++++++++++++++- pkg/credentialprovider/plugins.go | 8 +++++++ pkg/features/kube_features.go | 7 ++++++ 5 files changed, 55 insertions(+), 1 deletion(-) diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go index a0eef38594a..9537f6b82e8 100644 --- a/pkg/credentialprovider/aws/aws_credentials.go +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -159,12 +159,20 @@ func (p *ecrProvider) Provide(image string) credentialprovider.DockerConfig { // (see https://github.com/kubernetes/kubernetes/issues/92162) once.Do(func() { isEC2 = p.isEC2() + + if isEC2 && credentialprovider.AreLegacyCloudCredentialProvidersDisabled() { + klog.V(4).Infof("AWS credential provider is now disabled. Please refer to sig-cloud-provider for guidance on external credential provider integration for AWS") + } }) if !isEC2 { return credentialprovider.DockerConfig{} } + if credentialprovider.AreLegacyCloudCredentialProvidersDisabled() { + return credentialprovider.DockerConfig{} + } + if cfg, exists := p.getFromCache(parsed); exists { klog.V(3).Infof("Got ECR credentials from cache for %s", parsed.registry) return cfg diff --git a/pkg/credentialprovider/azure/azure_credentials.go b/pkg/credentialprovider/azure/azure_credentials.go index 2d9281482bf..035006e57bd 100644 --- a/pkg/credentialprovider/azure/azure_credentials.go +++ b/pkg/credentialprovider/azure/azure_credentials.go @@ -26,6 +26,7 @@ import ( "os" "regexp" "strings" + "sync" "time" "github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2019-05-01/containerregistry" @@ -52,6 +53,7 @@ const ( var ( containerRegistryUrls = []string{"*.azurecr.io", "*.azurecr.cn", "*.azurecr.de", "*.azurecr.us"} acrRE = regexp.MustCompile(`.*\.azurecr\.io|.*\.azurecr\.cn|.*\.azurecr\.de|.*\.azurecr\.us`) + warnOnce sync.Once ) // init registers the various means by which credentials may @@ -183,6 +185,13 @@ func (a *acrProvider) Enabled() bool { return false } + if credentialprovider.AreLegacyCloudCredentialProvidersDisabled() { + warnOnce.Do(func() { + klog.V(4).Infof("Azure credential provider is now disabled. Please refer to sig-cloud-provider for guidance on external credential provider integration for Azure") + }) + return false + } + f, err := os.Open(*a.file) if err != nil { klog.Errorf("Failed to load config from file: %s", *a.file) @@ -203,6 +212,7 @@ func (a *acrProvider) Enabled() bool { } a.registryClient = newAzRegistriesClient(a.config.SubscriptionID, a.environment.ResourceManagerEndpoint, a.servicePrincipalToken) + return true } diff --git a/pkg/credentialprovider/gcp/metadata.go b/pkg/credentialprovider/gcp/metadata.go index 8dd5f2afda9..d87bd5a9440 100644 --- a/pkg/credentialprovider/gcp/metadata.go +++ b/pkg/credentialprovider/gcp/metadata.go @@ -23,6 +23,7 @@ import ( "os/exec" "runtime" "strings" + "sync" "time" utilnet "k8s.io/apimachinery/pkg/util/net" @@ -55,6 +56,8 @@ var metadataHeader = &http.Header{ "Metadata-Flavor": []string{"Google"}, } +var warnOnce sync.Once + // init registers the various means by which credentials may // be resolved on GCP. func init() { @@ -142,7 +145,17 @@ func onGCEVM() bool { // Enabled implements DockerConfigProvider for all of the Google implementations. func (g *MetadataProvider) Enabled() bool { - return onGCEVM() + onGCE := onGCEVM() + if !onGCE { + return false + } + if credentialprovider.AreLegacyCloudCredentialProvidersDisabled() { + warnOnce.Do(func() { + klog.V(4).Infof("GCP credential provider is now disabled. Please refer to sig-cloud-provider for guidance on external credential provider integration for GCP") + }) + return false + } + return true } // Provide implements DockerConfigProvider @@ -187,6 +200,14 @@ func (g *ContainerRegistryProvider) Enabled() bool { if !onGCEVM() { return false } + + if credentialprovider.AreLegacyCloudCredentialProvidersDisabled() { + warnOnce.Do(func() { + klog.V(4).Infof("GCP credential provider is now disabled. Please refer to sig-cloud-provider for guidance on external credential provider integration for GCP") + }) + return false + } + // Given that we are on GCE, we should keep retrying until the metadata server responds. value := runWithBackoff(func() ([]byte, error) { value, err := gcpcredential.ReadURL(serviceAccounts, g.Client, metadataHeader) diff --git a/pkg/credentialprovider/plugins.go b/pkg/credentialprovider/plugins.go index 76051a9b4b5..a8e7a6858a0 100644 --- a/pkg/credentialprovider/plugins.go +++ b/pkg/credentialprovider/plugins.go @@ -21,7 +21,9 @@ import ( "sort" "sync" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" ) // All registered credential providers. @@ -44,6 +46,12 @@ func RegisterCredentialProvider(name string, provider DockerConfigProvider) { providers[name] = provider } +// AreLegacyCloudCredentialProvidersDisabled checks if the legacy in-tree cloud +// credential providers have been disabled. +func AreLegacyCloudCredentialProvidersDisabled() bool { + return utilfeature.DefaultFeatureGate.Enabled(features.DisableKubeletCloudCredentialProviders) +} + // NewDockerKeyring creates a DockerKeyring to use for resolving credentials, // which draws from the set of registered credential providers. func NewDockerKeyring() DockerKeyring { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index cd096d8eec8..32469b4c34f 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -595,6 +595,12 @@ const ( // Disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the `--cloud-provider` component flag. DisableCloudProviders featuregate.Feature = "DisableCloudProviders" + // owner: @andrewsykim + // alpha: v1.22 + // + // Disable in-tree functionality in kubelet to authenticate to cloud provider container registries for image pull credentials. + DisableKubeletCloudCredentialProviders featuregate.Feature = "DisableKubeletCloudCredentialProviders" + // owner: @zshihang // alpha: v1.20 // beta: v1.21 @@ -895,6 +901,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS CSIVolumeHealth: {Default: false, PreRelease: featuregate.Alpha}, WindowsHostProcessContainers: {Default: false, PreRelease: featuregate.Alpha}, DisableCloudProviders: {Default: false, PreRelease: featuregate.Alpha}, + DisableKubeletCloudCredentialProviders: {Default: false, PreRelease: featuregate.Alpha}, StatefulSetMinReadySeconds: {Default: false, PreRelease: featuregate.Alpha}, ExpandedDNSConfig: {Default: false, PreRelease: featuregate.Alpha}, SeccompDefault: {Default: false, PreRelease: featuregate.Alpha},