From 42fb89eb89b1f75e8dbd6052e903d932a7c77432 Mon Sep 17 00:00:00 2001 From: Kermit Alexander Date: Thu, 12 Nov 2020 01:19:48 +0000 Subject: [PATCH 1/3] Move config and provider code out of pkg/credentialprovider and into staging. --- cmd/kubelet/app/server.go | 5 +- pkg/credentialprovider/aws/aws_credentials.go | 21 +- .../azure/azure_credentials.go | 29 +- pkg/credentialprovider/default_provider.go | 52 ++++ pkg/credentialprovider/gcp/metadata.go | 260 +---------------- pkg/credentialprovider/gcp/metadata_test.go | 72 ++--- pkg/credentialprovider/keyring.go | 5 +- pkg/credentialprovider/keyring_test.go | 30 +- pkg/credentialprovider/plugins.go | 7 +- pkg/credentialprovider/secrets/secrets.go | 7 +- .../credentialconfig}/config.go | 8 +- .../credentialconfig}/config_test.go | 4 +- .../credentialconfig}/provider.go | 31 +- .../credentialconfig}/provider_test.go | 2 +- .../gce/gcpcredential/gcpcredential.go | 266 ++++++++++++++++++ 15 files changed, 429 insertions(+), 370 deletions(-) create mode 100644 pkg/credentialprovider/default_provider.go rename {pkg/credentialprovider => staging/src/k8s.io/cloud-provider/credentialconfig}/config.go (98%) rename {pkg/credentialprovider => staging/src/k8s.io/cloud-provider/credentialconfig}/config_test.go (99%) rename {pkg/credentialprovider => staging/src/k8s.io/cloud-provider/credentialconfig}/provider.go (74%) rename {pkg/credentialprovider => staging/src/k8s.io/cloud-provider/credentialconfig}/provider_test.go (98%) create mode 100644 staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/gcpcredential.go diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 6a60dc3206d..41ec2181e0c 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -59,6 +59,8 @@ import ( "k8s.io/client-go/util/connrotation" "k8s.io/client-go/util/keyutil" cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider/credentialconfig" + "k8s.io/component-base/cli/flag" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/configz" "k8s.io/component-base/featuregate" @@ -72,7 +74,6 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/capabilities" - "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet" kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" @@ -1116,7 +1117,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie AllowPrivileged: true, }) - credentialprovider.SetPreferredDockercfgPath(kubeServer.RootDirectory) + credentialconfig.SetPreferredDockercfgPath(kubeServer.RootDirectory) klog.V(2).Infof("Using root directory: %v", kubeServer.RootDirectory) if kubeDeps.OSInterface == nil { diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go index 8f85237e8f3..76bf7feee40 100644 --- a/pkg/credentialprovider/aws/aws_credentials.go +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" + "k8s.io/cloud-provider/credentialconfig" "k8s.io/component-base/version" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/credentialprovider" @@ -54,7 +55,7 @@ type ecrProvider struct { getterFactory tokenGetterFactory } -var _ credentialprovider.DockerConfigProvider = &ecrProvider{} +var _ credentialconfig.DockerConfigProvider = &ecrProvider{} func newECRProvider(getterFactory tokenGetterFactory) *ecrProvider { return &ecrProvider{ @@ -82,11 +83,11 @@ func (p *ecrProvider) Enabled() bool { // Provide returns a DockerConfig with credentials from the cache if they are // found, or from ECR -func (p *ecrProvider) Provide(image string) credentialprovider.DockerConfig { +func (p *ecrProvider) Provide(image string) credentialconfig.DockerConfig { parsed, err := parseRepoURL(image) if err != nil { klog.V(3).Info(err) - return credentialprovider.DockerConfig{} + return credentialconfig.DockerConfig{} } if cfg, exists := p.getFromCache(parsed); exists { @@ -98,15 +99,15 @@ func (p *ecrProvider) Provide(image string) credentialprovider.DockerConfig { cfg, err := p.getFromECR(parsed) if err != nil { klog.Errorf("error getting credentials from ECR for %s %v", parsed.registry, err) - return credentialprovider.DockerConfig{} + return credentialconfig.DockerConfig{} } klog.V(3).Infof("Got ECR credentials from ECR API for %s", parsed.registry) return cfg } // getFromCache attempts to get credentials from the cache -func (p *ecrProvider) getFromCache(parsed *parsedURL) (credentialprovider.DockerConfig, bool) { - cfg := credentialprovider.DockerConfig{} +func (p *ecrProvider) getFromCache(parsed *parsedURL) (credentialconfig.DockerConfig, bool) { + cfg := credentialconfig.DockerConfig{} obj, exists, err := p.cache.GetByKey(parsed.registry) if err != nil { @@ -124,8 +125,8 @@ func (p *ecrProvider) getFromCache(parsed *parsedURL) (credentialprovider.Docker } // getFromECR gets credentials from ECR since they are not in the cache -func (p *ecrProvider) getFromECR(parsed *parsedURL) (credentialprovider.DockerConfig, error) { - cfg := credentialprovider.DockerConfig{} +func (p *ecrProvider) getFromECR(parsed *parsedURL) (credentialconfig.DockerConfig, error) { + cfg := credentialconfig.DockerConfig{} getter, err := p.getterFactory.GetTokenGetterForRegion(parsed.region) if err != nil { return cfg, err @@ -260,7 +261,7 @@ func (p *ecrTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenI type cacheEntry struct { expiresAt time.Time - credentials credentialprovider.DockerConfigEntry + credentials credentialconfig.DockerConfigEntry registry string } @@ -275,7 +276,7 @@ func makeCacheEntry(data *ecr.AuthorizationData, registry string) (*cacheEntry, if len(parts) < 2 { return nil, errors.New("error getting username and password from authorization token") } - creds := credentialprovider.DockerConfigEntry{ + creds := credentialconfig.DockerConfigEntry{ Username: parts[0], Password: parts[1], Email: "not@val.id", // ECR doesn't care and Docker is about to obsolete it diff --git a/pkg/credentialprovider/azure/azure_credentials.go b/pkg/credentialprovider/azure/azure_credentials.go index 2d9281482bf..1fbd3705992 100644 --- a/pkg/credentialprovider/azure/azure_credentials.go +++ b/pkg/credentialprovider/azure/azure_credentials.go @@ -35,6 +35,7 @@ import ( "github.com/spf13/pflag" "k8s.io/client-go/tools/cache" + "k8s.io/cloud-provider/credentialconfig" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/legacy-cloud-providers/azure/auth" @@ -65,7 +66,7 @@ func init() { type cacheEntry struct { expiresAt time.Time - credentials credentialprovider.DockerConfigEntry + credentials credentialconfig.DockerConfigEntry registry string } @@ -122,7 +123,7 @@ func (az *azRegistriesClient) List(ctx context.Context) ([]containerregistry.Reg } // NewACRProvider parses the specified configFile and returns a DockerConfigProvider -func NewACRProvider(configFile *string) credentialprovider.DockerConfigProvider { +func NewACRProvider(configFile *string) credentialconfig.DockerConfigProvider { return &acrProvider{ file: configFile, cache: cache.NewExpirationStore(stringKeyFunc, &acrExpirationPolicy{}), @@ -207,8 +208,8 @@ func (a *acrProvider) Enabled() bool { } // getFromCache attempts to get credentials from the cache -func (a *acrProvider) getFromCache(loginServer string) (credentialprovider.DockerConfig, bool) { - cfg := credentialprovider.DockerConfig{} +func (a *acrProvider) getFromCache(loginServer string) (credentialconfig.DockerConfig, bool) { + cfg := credentialconfig.DockerConfig{} obj, exists, err := a.cache.GetByKey(loginServer) if err != nil { klog.Errorf("error getting ACR credentials from cache: %v", err) @@ -224,8 +225,8 @@ func (a *acrProvider) getFromCache(loginServer string) (credentialprovider.Docke } // getFromACR gets credentials from ACR since they are not in the cache -func (a *acrProvider) getFromACR(loginServer string) (credentialprovider.DockerConfig, error) { - cfg := credentialprovider.DockerConfig{} +func (a *acrProvider) getFromACR(loginServer string) (credentialconfig.DockerConfig, error) { + cfg := credentialconfig.DockerConfig{} cred, err := getACRDockerEntryFromARMToken(a, loginServer) if err != nil { return cfg, err @@ -243,14 +244,14 @@ func (a *acrProvider) getFromACR(loginServer string) (credentialprovider.DockerC return cfg, nil } -func (a *acrProvider) Provide(image string) credentialprovider.DockerConfig { +func (a *acrProvider) Provide(image string) credentialconfig.DockerConfig { loginServer := a.parseACRLoginServerFromImage(image) if loginServer == "" { klog.V(2).Infof("image(%s) is not from ACR, return empty authentication", image) - return credentialprovider.DockerConfig{} + return credentialconfig.DockerConfig{} } - cfg := credentialprovider.DockerConfig{} + cfg := credentialconfig.DockerConfig{} if a.config != nil && a.config.UseManagedIdentityExtension { var exists bool cfg, exists = a.getFromCache(loginServer) @@ -267,7 +268,7 @@ func (a *acrProvider) Provide(image string) credentialprovider.DockerConfig { } else { // Add our entry for each of the supported container registry URLs for _, url := range containerRegistryUrls { - cred := &credentialprovider.DockerConfigEntry{ + cred := &credentialconfig.DockerConfigEntry{ Username: a.config.AADClientID, Password: a.config.AADClientSecret, Email: dummyRegistryEmail, @@ -288,7 +289,7 @@ func (a *acrProvider) Provide(image string) credentialprovider.DockerConfig { } if !hasBeenAdded { - cred := &credentialprovider.DockerConfigEntry{ + cred := &credentialconfig.DockerConfigEntry{ Username: a.config.AADClientID, Password: a.config.AADClientSecret, Email: dummyRegistryEmail, @@ -299,7 +300,7 @@ func (a *acrProvider) Provide(image string) credentialprovider.DockerConfig { } // add ACR anonymous repo support: use empty username and password for anonymous access - defaultConfigEntry := credentialprovider.DockerConfigEntry{ + defaultConfigEntry := credentialconfig.DockerConfigEntry{ Username: "", Password: "", Email: dummyRegistryEmail, @@ -312,7 +313,7 @@ func getLoginServer(registry containerregistry.Registry) string { return *(*registry.RegistryProperties).LoginServer } -func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credentialprovider.DockerConfigEntry, error) { +func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credentialconfig.DockerConfigEntry, error) { // Run EnsureFresh to make sure the token is valid and does not expire if err := a.servicePrincipalToken.EnsureFresh(); err != nil { klog.Errorf("Failed to ensure fresh service principal token: %v", err) @@ -336,7 +337,7 @@ func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credent } klog.V(4).Infof("adding ACR docker config entry for: %s", loginServer) - return &credentialprovider.DockerConfigEntry{ + return &credentialconfig.DockerConfigEntry{ Username: dockerTokenLoginUsernameGUID, Password: registryRefreshToken, Email: dummyRegistryEmail, diff --git a/pkg/credentialprovider/default_provider.go b/pkg/credentialprovider/default_provider.go new file mode 100644 index 00000000000..f85ef49c83d --- /dev/null +++ b/pkg/credentialprovider/default_provider.go @@ -0,0 +1,52 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package credentialprovider + +import ( + "k8s.io/cloud-provider/credentialconfig" + "k8s.io/klog/v2" + "os" + "time" +) + +// A DockerConfigProvider that simply reads the .dockercfg file +type defaultDockerConfigProvider struct{} + +// init registers our default provider, which simply reads the .dockercfg file. +func init() { + RegisterCredentialProvider(".dockercfg", + &credentialconfig.CachingDockerConfigProvider{ + Provider: &defaultDockerConfigProvider{}, + Lifetime: 5 * time.Minute, + }) +} + +// Enabled implements dockerConfigProvider +func (d *defaultDockerConfigProvider) Enabled() bool { + return true +} + +// Provide implements dockerConfigProvider +func (d *defaultDockerConfigProvider) Provide(image string) credentialconfig.DockerConfig { + // Read the standard Docker credentials from .dockercfg + if cfg, err := credentialconfig.ReadDockerConfigFile(); err == nil { + return cfg + } else if !os.IsNotExist(err) { + klog.V(4).Infof("Unable to parse Docker config file: %v", err) + } + return credentialconfig.DockerConfig{} +} diff --git a/pkg/credentialprovider/gcp/metadata.go b/pkg/credentialprovider/gcp/metadata.go index c7f0b994c8e..a13cd4a42cb 100644 --- a/pkg/credentialprovider/gcp/metadata.go +++ b/pkg/credentialprovider/gcp/metadata.go @@ -17,70 +17,15 @@ limitations under the License. package gcp import ( - "encoding/json" - "io/ioutil" "net/http" - "os/exec" - "runtime" - "strings" "time" utilnet "k8s.io/apimachinery/pkg/util/net" - "k8s.io/klog/v2" + "k8s.io/cloud-provider/credentialconfig" "k8s.io/kubernetes/pkg/credentialprovider" + "k8s.io/legacy-cloud-providers/gce/gcpcredential" ) -const ( - metadataURL = "http://metadata.google.internal./computeMetadata/v1/" - metadataAttributes = metadataURL + "instance/attributes/" - dockerConfigKey = metadataAttributes + "google-dockercfg" - dockerConfigURLKey = metadataAttributes + "google-dockercfg-url" - serviceAccounts = metadataURL + "instance/service-accounts/" - metadataScopes = metadataURL + "instance/service-accounts/default/scopes" - metadataToken = metadataURL + "instance/service-accounts/default/token" - metadataEmail = metadataURL + "instance/service-accounts/default/email" - storageScopePrefix = "https://www.googleapis.com/auth/devstorage" - cloudPlatformScopePrefix = "https://www.googleapis.com/auth/cloud-platform" - defaultServiceAccount = "default/" -) - -// Product file path that contains the cloud service name. -// This is a variable instead of a const to enable testing. -var gceProductNameFile = "/sys/class/dmi/id/product_name" - -// For these urls, the parts of the host name can be glob, for example '*.gcr.io" will match -// "foo.gcr.io" and "bar.gcr.io". -var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io", "*.pkg.dev"} - -var metadataHeader = &http.Header{ - "Metadata-Flavor": []string{"Google"}, -} - -// A DockerConfigProvider that reads its configuration from Google -// Compute Engine metadata. -type metadataProvider struct { - Client *http.Client -} - -// A DockerConfigProvider that reads its configuration from a specific -// Google Compute Engine metadata key: 'google-dockercfg'. -type dockerConfigKeyProvider struct { - metadataProvider -} - -// A DockerConfigProvider that reads its configuration from a URL read from -// a specific Google Compute Engine metadata key: 'google-dockercfg-url'. -type dockerConfigURLKeyProvider struct { - metadataProvider -} - -// A DockerConfigProvider that provides a dockercfg with: -// Username: "_token" -// Password: "{access token from metadata}" -type containerRegistryProvider struct { - metadataProvider -} - // init registers the various means by which credentials may // be resolved on GCP. func init() { @@ -91,17 +36,17 @@ func init() { Timeout: metadataHTTPClientTimeout, } credentialprovider.RegisterCredentialProvider("google-dockercfg", - &credentialprovider.CachingDockerConfigProvider{ - Provider: &dockerConfigKeyProvider{ - metadataProvider{Client: httpClient}, + &credentialconfig.CachingDockerConfigProvider{ + Provider: &gcpcredential.DockerConfigKeyProvider{ + gcpcredential.MetadataProvider{Client: httpClient}, }, Lifetime: 60 * time.Second, }) credentialprovider.RegisterCredentialProvider("google-dockercfg-url", - &credentialprovider.CachingDockerConfigProvider{ - Provider: &dockerConfigURLKeyProvider{ - metadataProvider{Client: httpClient}, + &credentialconfig.CachingDockerConfigProvider{ + Provider: &gcpcredential.DockerConfigURLKeyProvider{ + gcpcredential.MetadataProvider{Client: httpClient}, }, Lifetime: 60 * time.Second, }) @@ -109,192 +54,7 @@ func init() { credentialprovider.RegisterCredentialProvider("google-container-registry", // Never cache this. The access token is already // cached by the metadata service. - &containerRegistryProvider{ - metadataProvider{Client: httpClient}, + &gcpcredential.ContainerRegistryProvider{ + gcpcredential.MetadataProvider{Client: httpClient}, }) } - -// Returns true if it finds a local GCE VM. -// Looks at a product file that is an undocumented API. -func onGCEVM() bool { - var name string - - if runtime.GOOS == "windows" { - data, err := exec.Command("wmic", "computersystem", "get", "model").Output() - if err != nil { - return false - } - fields := strings.Split(strings.TrimSpace(string(data)), "\r\n") - if len(fields) != 2 { - klog.V(2).Infof("Received unexpected value retrieving system model: %q", string(data)) - return false - } - name = fields[1] - } else { - data, err := ioutil.ReadFile(gceProductNameFile) - if err != nil { - klog.V(2).Infof("Error while reading product_name: %v", err) - return false - } - name = strings.TrimSpace(string(data)) - } - return name == "Google" || name == "Google Compute Engine" -} - -// Enabled implements DockerConfigProvider for all of the Google implementations. -func (g *metadataProvider) Enabled() bool { - return onGCEVM() -} - -// Provide implements DockerConfigProvider -func (g *dockerConfigKeyProvider) Provide(image string) credentialprovider.DockerConfig { - // Read the contents of the google-dockercfg metadata key and - // parse them as an alternate .dockercfg - if cfg, err := credentialprovider.ReadDockerConfigFileFromURL(dockerConfigKey, g.Client, metadataHeader); err != nil { - klog.Errorf("while reading 'google-dockercfg' metadata: %v", err) - } else { - return cfg - } - - return credentialprovider.DockerConfig{} -} - -// Provide implements DockerConfigProvider -func (g *dockerConfigURLKeyProvider) Provide(image string) credentialprovider.DockerConfig { - // Read the contents of the google-dockercfg-url key and load a .dockercfg from there - if url, err := credentialprovider.ReadURL(dockerConfigURLKey, g.Client, metadataHeader); err != nil { - klog.Errorf("while reading 'google-dockercfg-url' metadata: %v", err) - } else { - if strings.HasPrefix(string(url), "http") { - if cfg, err := credentialprovider.ReadDockerConfigFileFromURL(string(url), g.Client, nil); err != nil { - klog.Errorf("while reading 'google-dockercfg-url'-specified url: %s, %v", string(url), err) - } else { - return cfg - } - } else { - // TODO(mattmoor): support reading alternate scheme URLs (e.g. gs:// or s3://) - klog.Errorf("Unsupported URL scheme: %s", string(url)) - } - } - - return credentialprovider.DockerConfig{} -} - -// runWithBackoff runs input function `f` with an exponential backoff. -// Note that this method can block indefinitely. -func runWithBackoff(f func() ([]byte, error)) []byte { - var backoff = 100 * time.Millisecond - const maxBackoff = time.Minute - for { - value, err := f() - if err == nil { - return value - } - time.Sleep(backoff) - backoff = backoff * 2 - if backoff > maxBackoff { - backoff = maxBackoff - } - } -} - -// Enabled implements a special metadata-based check, which verifies the -// storage scope is available on the GCE VM. -// If running on a GCE VM, check if 'default' service account exists. -// If it does not exist, assume that registry is not enabled. -// If default service account exists, check if relevant scopes exist in the default service account. -// The metadata service can become temporarily inaccesible. Hence all requests to the metadata -// service will be retried until the metadata server returns a `200`. -// It is expected that "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/" will return a `200` -// and "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/scopes" will also return `200`. -// More information on metadata service can be found here - https://cloud.google.com/compute/docs/storing-retrieving-metadata -func (g *containerRegistryProvider) Enabled() bool { - if !onGCEVM() { - return false - } - // Given that we are on GCE, we should keep retrying until the metadata server responds. - value := runWithBackoff(func() ([]byte, error) { - value, err := credentialprovider.ReadURL(serviceAccounts, g.Client, metadataHeader) - if err != nil { - klog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err) - } - return value, err - }) - // We expect the service account to return a list of account directories separated by newlines, e.g., - // sv-account-name1/ - // sv-account-name2/ - // ref: https://cloud.google.com/compute/docs/storing-retrieving-metadata - defaultServiceAccountExists := false - for _, sa := range strings.Split(string(value), "\n") { - if strings.TrimSpace(sa) == defaultServiceAccount { - defaultServiceAccountExists = true - break - } - } - if !defaultServiceAccountExists { - klog.V(2).Infof("'default' service account does not exist. Found following service accounts: %q", string(value)) - return false - } - url := metadataScopes + "?alt=json" - value = runWithBackoff(func() ([]byte, error) { - value, err := credentialprovider.ReadURL(url, g.Client, metadataHeader) - if err != nil { - klog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err) - } - return value, err - }) - var scopes []string - if err := json.Unmarshal(value, &scopes); err != nil { - klog.Errorf("Failed to unmarshal scopes: %v", err) - return false - } - for _, v := range scopes { - // cloudPlatformScope implies storage scope. - if strings.HasPrefix(v, storageScopePrefix) || strings.HasPrefix(v, cloudPlatformScopePrefix) { - return true - } - } - klog.Warningf("Google container registry is disabled, no storage scope is available: %s", value) - return false -} - -// tokenBlob is used to decode the JSON blob containing an access token -// that is returned by GCE metadata. -type tokenBlob struct { - AccessToken string `json:"access_token"` -} - -// Provide implements DockerConfigProvider -func (g *containerRegistryProvider) Provide(image string) credentialprovider.DockerConfig { - cfg := credentialprovider.DockerConfig{} - - tokenJSONBlob, err := credentialprovider.ReadURL(metadataToken, g.Client, metadataHeader) - if err != nil { - klog.Errorf("while reading access token endpoint: %v", err) - return cfg - } - - email, err := credentialprovider.ReadURL(metadataEmail, g.Client, metadataHeader) - if err != nil { - klog.Errorf("while reading email endpoint: %v", err) - return cfg - } - - var parsedBlob tokenBlob - if err := json.Unmarshal([]byte(tokenJSONBlob), &parsedBlob); err != nil { - klog.Errorf("while parsing json blob %s: %v", tokenJSONBlob, err) - return cfg - } - - entry := credentialprovider.DockerConfigEntry{ - Username: "_token", - Password: parsedBlob.AccessToken, - Email: string(email), - } - - // Add our entry for each of the supported container registry URLs - for _, k := range containerRegistryUrls { - cfg[k] = entry - } - return cfg -} diff --git a/pkg/credentialprovider/gcp/metadata_test.go b/pkg/credentialprovider/gcp/metadata_test.go index 29ca3e2e4f0..5b4af49415a 100644 --- a/pkg/credentialprovider/gcp/metadata_test.go +++ b/pkg/credentialprovider/gcp/metadata_test.go @@ -30,7 +30,9 @@ import ( "testing" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/cloud-provider/credentialconfig" "k8s.io/kubernetes/pkg/credentialprovider" + "k8s.io/legacy-cloud-providers/gce/gcpcredential" ) func createProductNameFile() (string, error) { @@ -55,17 +57,17 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) { }`, registryURL, email, auth) var err error - gceProductNameFile, err = createProductNameFile() + gcpcredential.GCEProductNameFile, err = createProductNameFile() if err != nil { t.Errorf("failed to create gce product name file: %v", err) } - defer os.Remove(gceProductNameFile) + defer os.Remove(gcpcredential.GCEProductNameFile) const probeEndpoint = "/computeMetadata/v1/" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Only serve the one metadata key. if probeEndpoint == r.URL.Path { w.WriteHeader(http.StatusOK) - } else if strings.HasSuffix(dockerConfigKey, r.URL.Path) { + } else if strings.HasSuffix(gcpcredential.DockerConfigKey, r.URL.Path) { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/json") fmt.Fprintln(w, sampleDockerConfig) @@ -83,8 +85,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) { }) keyring := &credentialprovider.BasicDockerKeyring{} - provider := &dockerConfigKeyProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + provider := &gcpcredential.DockerConfigKeyProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, } if !provider.Enabled() { @@ -128,11 +130,11 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) { }`, registryURL, email, auth) var err error - gceProductNameFile, err = createProductNameFile() + gcpcredential.GCEProductNameFile, err = createProductNameFile() if err != nil { t.Errorf("failed to create gce product name file: %v", err) } - defer os.Remove(gceProductNameFile) + defer os.Remove(gcpcredential.GCEProductNameFile) const probeEndpoint = "/computeMetadata/v1/" const valueEndpoint = "/my/value" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -143,7 +145,7 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/json") fmt.Fprintln(w, sampleDockerConfig) - } else if strings.HasSuffix(dockerConfigURLKey, r.URL.Path) { + } else if strings.HasSuffix(gcpcredential.DockerConfigURLKey, r.URL.Path) { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/text") fmt.Fprint(w, "http://foo.bar.com"+valueEndpoint) @@ -161,8 +163,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) { }) keyring := &credentialprovider.BasicDockerKeyring{} - provider := &dockerConfigURLKeyProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + provider := &gcpcredential.DockerConfigURLKeyProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, } if !provider.Enabled() { @@ -197,7 +199,7 @@ func TestContainerRegistryBasics(t *testing.T) { for _, registryURL := range registryURLs { t.Run(registryURL, func(t *testing.T) { email := "1234@project.gserviceaccount.com" - token := &tokenBlob{AccessToken: "ya26.lots-of-indiscernible-garbage"} // Fake value for testing. + token := &gcpcredential.TokenBlob{AccessToken: "ya26.lots-of-indiscernible-garbage"} // Fake value for testing. const ( serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/" @@ -207,18 +209,18 @@ func TestContainerRegistryBasics(t *testing.T) { tokenEndpoint = defaultEndpoint + "token" ) var err error - gceProductNameFile, err = createProductNameFile() + gcpcredential.GCEProductNameFile, err = createProductNameFile() if err != nil { t.Errorf("failed to create gce product name file: %v", err) } - defer os.Remove(gceProductNameFile) + defer os.Remove(gcpcredential.GCEProductNameFile) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Only serve the URL key and the value endpoint if scopeEndpoint == r.URL.Path { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/json") - fmt.Fprintf(w, `["%s.read_write"]`, storageScopePrefix) + fmt.Fprintf(w, `["%s.read_write"]`, gcpcredential.StorageScopePrefix) } else if emailEndpoint == r.URL.Path { w.WriteHeader(http.StatusOK) fmt.Fprint(w, email) @@ -247,8 +249,8 @@ func TestContainerRegistryBasics(t *testing.T) { }) keyring := &credentialprovider.BasicDockerKeyring{} - provider := &containerRegistryProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + provider := &gcpcredential.ContainerRegistryProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, } if !provider.Enabled() { @@ -301,11 +303,11 @@ func TestContainerRegistryNoServiceAccount(t *testing.T) { defer server.Close() var err error - gceProductNameFile, err = createProductNameFile() + gcpcredential.GCEProductNameFile, err = createProductNameFile() if err != nil { t.Errorf("failed to create gce product name file: %v", err) } - defer os.Remove(gceProductNameFile) + defer os.Remove(gcpcredential.GCEProductNameFile) // Make a transport that reroutes all traffic to the example server transport := utilnet.SetTransportDefaults(&http.Transport{ @@ -314,8 +316,8 @@ func TestContainerRegistryNoServiceAccount(t *testing.T) { }, }) - provider := &containerRegistryProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + provider := &gcpcredential.ContainerRegistryProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, } if provider.Enabled() { @@ -345,11 +347,11 @@ func TestContainerRegistryNoStorageScope(t *testing.T) { defer server.Close() var err error - gceProductNameFile, err = createProductNameFile() + gcpcredential.GCEProductNameFile, err = createProductNameFile() if err != nil { t.Errorf("failed to create gce product name file: %v", err) } - defer os.Remove(gceProductNameFile) + defer os.Remove(gcpcredential.GCEProductNameFile) // Make a transport that reroutes all traffic to the example server transport := utilnet.SetTransportDefaults(&http.Transport{ @@ -358,8 +360,8 @@ func TestContainerRegistryNoStorageScope(t *testing.T) { }, }) - provider := &containerRegistryProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + provider := &gcpcredential.ContainerRegistryProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, } if provider.Enabled() { @@ -390,11 +392,11 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) { defer server.Close() var err error - gceProductNameFile, err = createProductNameFile() + gcpcredential.GCEProductNameFile, err = createProductNameFile() if err != nil { t.Errorf("failed to create gce product name file: %v", err) } - defer os.Remove(gceProductNameFile) + defer os.Remove(gcpcredential.GCEProductNameFile) // Make a transport that reroutes all traffic to the example server transport := utilnet.SetTransportDefaults(&http.Transport{ @@ -403,8 +405,8 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) { }, }) - provider := &containerRegistryProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + provider := &gcpcredential.ContainerRegistryProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, } if !provider.Enabled() { @@ -425,15 +427,15 @@ func TestAllProvidersNoMetadata(t *testing.T) { }, }) - providers := []credentialprovider.DockerConfigProvider{ - &dockerConfigKeyProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + providers := []credentialconfig.DockerConfigProvider{ + &gcpcredential.DockerConfigKeyProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, }, - &dockerConfigURLKeyProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + &gcpcredential.DockerConfigURLKeyProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, }, - &containerRegistryProvider{ - metadataProvider{Client: &http.Client{Transport: transport}}, + &gcpcredential.ContainerRegistryProvider{ + gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, }, } diff --git a/pkg/credentialprovider/keyring.go b/pkg/credentialprovider/keyring.go index 5ce467f9a8b..750bcdd26e8 100644 --- a/pkg/credentialprovider/keyring.go +++ b/pkg/credentialprovider/keyring.go @@ -26,6 +26,7 @@ import ( "k8s.io/klog/v2" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/cloud-provider/credentialconfig" ) // DockerKeyring tracks a set of docker registry credentials, maintaining a @@ -48,7 +49,7 @@ type BasicDockerKeyring struct { // providersDockerKeyring is an implementation of DockerKeyring that // materializes its dockercfg based on a set of dockerConfigProviders. type providersDockerKeyring struct { - Providers []DockerConfigProvider + Providers []credentialconfig.DockerConfigProvider } // AuthConfig contains authorization information for connecting to a Registry @@ -74,7 +75,7 @@ type AuthConfig struct { } // Add add some docker config in basic docker keyring -func (dk *BasicDockerKeyring) Add(cfg DockerConfig) { +func (dk *BasicDockerKeyring) Add(cfg credentialconfig.DockerConfig) { if dk.index == nil { dk.index = make([]string, 0) dk.creds = make(map[string][]AuthConfig) diff --git a/pkg/credentialprovider/keyring_test.go b/pkg/credentialprovider/keyring_test.go index 3dd7bc2410d..dfbbf8a1275 100644 --- a/pkg/credentialprovider/keyring_test.go +++ b/pkg/credentialprovider/keyring_test.go @@ -21,6 +21,8 @@ import ( "fmt" "reflect" "testing" + + "k8s.io/cloud-provider/credentialconfig" ) func TestURLsMatch(t *testing.T) { @@ -203,7 +205,7 @@ func TestDockerKeyringForGlob(t *testing.T) { }`, test.globURL, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -271,7 +273,7 @@ func TestKeyringMiss(t *testing.T) { }`, test.globURL, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -299,7 +301,7 @@ func TestKeyringMissWithDockerHubCredentials(t *testing.T) { }`, url, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -325,7 +327,7 @@ func TestKeyringHitWithUnqualifiedDockerHub(t *testing.T) { }`, url, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -366,7 +368,7 @@ func TestKeyringHitWithUnqualifiedLibraryDockerHub(t *testing.T) { }`, url, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -407,7 +409,7 @@ func TestKeyringHitWithQualifiedDockerHub(t *testing.T) { }`, url, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -464,9 +466,9 @@ func (d *testProvider) Enabled() bool { } // Provide implements dockerConfigProvider -func (d *testProvider) Provide(image string) DockerConfig { +func (d *testProvider) Provide(image string) credentialconfig.DockerConfig { d.Count++ - return DockerConfig{} + return credentialconfig.DockerConfig{} } func TestProvidersDockerKeyring(t *testing.T) { @@ -474,7 +476,7 @@ func TestProvidersDockerKeyring(t *testing.T) { Count: 0, } keyring := &providersDockerKeyring{ - Providers: []DockerConfigProvider{ + Providers: []credentialconfig.DockerConfigProvider{ provider, }, } @@ -510,13 +512,13 @@ func TestDockerKeyringLookup(t *testing.T) { } dk := &BasicDockerKeyring{} - dk.Add(DockerConfig{ - "bar.example.com/pong": DockerConfigEntry{ + dk.Add(credentialconfig.DockerConfig{ + "bar.example.com/pong": credentialconfig.DockerConfigEntry{ Username: grace.Username, Password: grace.Password, Email: grace.Email, }, - "bar.example.com": DockerConfigEntry{ + "bar.example.com": credentialconfig.DockerConfigEntry{ Username: ada.Username, Password: ada.Password, Email: ada.Email, @@ -571,8 +573,8 @@ func TestIssue3797(t *testing.T) { } dk := &BasicDockerKeyring{} - dk.Add(DockerConfig{ - "https://quay.io/v1/": DockerConfigEntry{ + dk.Add(credentialconfig.DockerConfig{ + "https://quay.io/v1/": credentialconfig.DockerConfigEntry{ Username: rex.Username, Password: rex.Password, Email: rex.Email, diff --git a/pkg/credentialprovider/plugins.go b/pkg/credentialprovider/plugins.go index 76051a9b4b5..ed65aeec4c1 100644 --- a/pkg/credentialprovider/plugins.go +++ b/pkg/credentialprovider/plugins.go @@ -21,19 +21,20 @@ import ( "sort" "sync" + "k8s.io/cloud-provider/credentialconfig" "k8s.io/klog/v2" ) // All registered credential providers. var providersMutex sync.Mutex -var providers = make(map[string]DockerConfigProvider) +var providers = make(map[string]credentialconfig.DockerConfigProvider) // RegisterCredentialProvider is called by provider implementations on // initialization to register themselves, like so: // func init() { // RegisterCredentialProvider("name", &myProvider{...}) // } -func RegisterCredentialProvider(name string, provider DockerConfigProvider) { +func RegisterCredentialProvider(name string, provider credentialconfig.DockerConfigProvider) { providersMutex.Lock() defer providersMutex.Unlock() _, found := providers[name] @@ -48,7 +49,7 @@ func RegisterCredentialProvider(name string, provider DockerConfigProvider) { // which draws from the set of registered credential providers. func NewDockerKeyring() DockerKeyring { keyring := &providersDockerKeyring{ - Providers: make([]DockerConfigProvider, 0), + Providers: make([]credentialconfig.DockerConfigProvider, 0), } keys := reflect.ValueOf(providers).MapKeys() diff --git a/pkg/credentialprovider/secrets/secrets.go b/pkg/credentialprovider/secrets/secrets.go index 4e7c220b397..fb4664896ba 100644 --- a/pkg/credentialprovider/secrets/secrets.go +++ b/pkg/credentialprovider/secrets/secrets.go @@ -20,6 +20,7 @@ import ( "encoding/json" "k8s.io/api/core/v1" + "k8s.io/cloud-provider/credentialconfig" "k8s.io/kubernetes/pkg/credentialprovider" ) @@ -27,17 +28,17 @@ import ( // then a DockerKeyring is built based on every hit and unioned with the defaultKeyring. // If they do not, then the default keyring is returned func MakeDockerKeyring(passedSecrets []v1.Secret, defaultKeyring credentialprovider.DockerKeyring) (credentialprovider.DockerKeyring, error) { - passedCredentials := []credentialprovider.DockerConfig{} + passedCredentials := []credentialconfig.DockerConfig{} for _, passedSecret := range passedSecrets { if dockerConfigJSONBytes, dockerConfigJSONExists := passedSecret.Data[v1.DockerConfigJsonKey]; (passedSecret.Type == v1.SecretTypeDockerConfigJson) && dockerConfigJSONExists && (len(dockerConfigJSONBytes) > 0) { - dockerConfigJSON := credentialprovider.DockerConfigJSON{} + dockerConfigJSON := credentialconfig.DockerConfigJSON{} if err := json.Unmarshal(dockerConfigJSONBytes, &dockerConfigJSON); err != nil { return nil, err } passedCredentials = append(passedCredentials, dockerConfigJSON.Auths) } else if dockercfgBytes, dockercfgExists := passedSecret.Data[v1.DockerConfigKey]; (passedSecret.Type == v1.SecretTypeDockercfg) && dockercfgExists && (len(dockercfgBytes) > 0) { - dockercfg := credentialprovider.DockerConfig{} + dockercfg := credentialconfig.DockerConfig{} if err := json.Unmarshal(dockercfgBytes, &dockercfg); err != nil { return nil, err } diff --git a/pkg/credentialprovider/config.go b/staging/src/k8s.io/cloud-provider/credentialconfig/config.go similarity index 98% rename from pkg/credentialprovider/config.go rename to staging/src/k8s.io/cloud-provider/credentialconfig/config.go index b9edb4c39e8..838840a2a5f 100644 --- a/pkg/credentialprovider/config.go +++ b/staging/src/k8s.io/cloud-provider/credentialconfig/config.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package credentialprovider +package credentialconfig import ( "encoding/base64" @@ -116,7 +116,7 @@ func ReadDockercfgFile(searchPaths []string) (cfg DockerConfig, err error) { klog.V(4).Infof("while trying to read %s: %v", absDockerConfigFileLocation, err) continue } - cfg, err := readDockerConfigFileFromBytes(contents) + cfg, err := ReadDockerConfigFileFromBytes(contents) if err != nil { klog.V(4).Infof("couldn't get the config from %q contents: %v", absDockerConfigFileLocation, err) continue @@ -226,13 +226,13 @@ func ReadURL(url string, client *http.Client, header *http.Header) (body []byte, // ReadDockerConfigFileFromURL read a docker config file from the given url func ReadDockerConfigFileFromURL(url string, client *http.Client, header *http.Header) (cfg DockerConfig, err error) { if contents, err := ReadURL(url, client, header); err == nil { - return readDockerConfigFileFromBytes(contents) + return ReadDockerConfigFileFromBytes(contents) } return nil, err } -func readDockerConfigFileFromBytes(contents []byte) (cfg DockerConfig, err error) { +func ReadDockerConfigFileFromBytes(contents []byte) (cfg DockerConfig, err error) { if err = json.Unmarshal(contents, &cfg); err != nil { return nil, errors.New("error occurred while trying to unmarshal json") } diff --git a/pkg/credentialprovider/config_test.go b/staging/src/k8s.io/cloud-provider/credentialconfig/config_test.go similarity index 99% rename from pkg/credentialprovider/config_test.go rename to staging/src/k8s.io/cloud-provider/credentialconfig/config_test.go index 0adc5053bf8..8a5aec0a52a 100644 --- a/pkg/credentialprovider/config_test.go +++ b/staging/src/k8s.io/cloud-provider/credentialconfig/config_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package credentialprovider +package credentialconfig import ( "encoding/base64" @@ -338,7 +338,7 @@ func TestReadDockerConfigFileFromBytes(t *testing.T) { } for _, tc := range testCases { - cfg, err := readDockerConfigFileFromBytes(tc.input) + cfg, err := ReadDockerConfigFileFromBytes(tc.input) if err != nil && !tc.errorExpected { t.Fatalf("Error was not expected: %v", err) } diff --git a/pkg/credentialprovider/provider.go b/staging/src/k8s.io/cloud-provider/credentialconfig/provider.go similarity index 74% rename from pkg/credentialprovider/provider.go rename to staging/src/k8s.io/cloud-provider/credentialconfig/provider.go index d575ca9155f..37943fda833 100644 --- a/pkg/credentialprovider/provider.go +++ b/staging/src/k8s.io/cloud-provider/credentialconfig/provider.go @@ -14,10 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -package credentialprovider +package credentialconfig import ( - "os" "reflect" "sync" "time" @@ -39,18 +38,6 @@ type DockerConfigProvider interface { Provide(image string) DockerConfig } -// A DockerConfigProvider that simply reads the .dockercfg file -type defaultDockerConfigProvider struct{} - -// init registers our default provider, which simply reads the .dockercfg file. -func init() { - RegisterCredentialProvider(".dockercfg", - &CachingDockerConfigProvider{ - Provider: &defaultDockerConfigProvider{}, - Lifetime: 5 * time.Minute, - }) -} - // CachingDockerConfigProvider implements DockerConfigProvider by composing // with another DockerConfigProvider and caching the DockerConfig it provides // for a pre-specified lifetime. @@ -68,22 +55,6 @@ type CachingDockerConfigProvider struct { mu sync.Mutex } -// Enabled implements dockerConfigProvider -func (d *defaultDockerConfigProvider) Enabled() bool { - return true -} - -// Provide implements dockerConfigProvider -func (d *defaultDockerConfigProvider) Provide(image string) DockerConfig { - // Read the standard Docker credentials from .dockercfg - if cfg, err := ReadDockerConfigFile(); err == nil { - return cfg - } else if !os.IsNotExist(err) { - klog.V(4).Infof("Unable to parse Docker config file: %v", err) - } - return DockerConfig{} -} - // Enabled implements dockerConfigProvider func (d *CachingDockerConfigProvider) Enabled() bool { return d.Provider.Enabled() diff --git a/pkg/credentialprovider/provider_test.go b/staging/src/k8s.io/cloud-provider/credentialconfig/provider_test.go similarity index 98% rename from pkg/credentialprovider/provider_test.go rename to staging/src/k8s.io/cloud-provider/credentialconfig/provider_test.go index 44a2f581976..254af4bfc42 100644 --- a/pkg/credentialprovider/provider_test.go +++ b/staging/src/k8s.io/cloud-provider/credentialconfig/provider_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package credentialprovider +package credentialconfig import ( "testing" diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/gcpcredential.go b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/gcpcredential.go new file mode 100644 index 00000000000..1105a339067 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/gcpcredential.go @@ -0,0 +1,266 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcpcredential + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "os/exec" + "runtime" + "strings" + "time" + + "k8s.io/cloud-provider/credentialconfig" + "k8s.io/klog/v2" +) + +const ( + metadataURL = "http://metadata.google.internal./computeMetadata/v1/" + metadataAttributes = metadataURL + "instance/attributes/" + DockerConfigKey = metadataAttributes + "google-dockercfg" + DockerConfigURLKey = metadataAttributes + "google-dockercfg-url" + serviceAccounts = metadataURL + "instance/service-accounts/" + metadataScopes = metadataURL + "instance/service-accounts/default/scopes" + metadataToken = metadataURL + "instance/service-accounts/default/token" + metadataEmail = metadataURL + "instance/service-accounts/default/email" + StorageScopePrefix = "https://www.googleapis.com/auth/devstorage" + cloudPlatformScopePrefix = "https://www.googleapis.com/auth/cloud-platform" + defaultServiceAccount = "default/" +) + +// Product file path that contains the cloud service name. +// This is a variable instead of a const to enable testing. +var GCEProductNameFile = "/sys/class/dmi/id/product_name" + +// For these urls, the parts of the host name can be glob, for example '*.gcr.io" will match +// "foo.gcr.io" and "bar.gcr.io". +var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io", "*.pkg.dev"} + +var metadataHeader = &http.Header{ + "Metadata-Flavor": []string{"Google"}, +} + +// A DockerConfigProvider that reads its configuration from Google +// Compute Engine metadata. +type MetadataProvider struct { + Client *http.Client +} + +// A DockerConfigProvider that reads its configuration from a specific +// Google Compute Engine metadata key: 'google-dockercfg'. +type DockerConfigKeyProvider struct { + MetadataProvider +} + +// A DockerConfigProvider that reads its configuration from a URL read from +// a specific Google Compute Engine metadata key: 'google-dockercfg-url'. +type DockerConfigURLKeyProvider struct { + MetadataProvider +} + +// A DockerConfigProvider that provides a dockercfg with: +// Username: "_token" +// Password: "{access token from metadata}" +type ContainerRegistryProvider struct { + MetadataProvider +} + +// Returns true if it finds a local GCE VM. +// Looks at a product file that is an undocumented API. +func onGCEVM() bool { + var name string + + if runtime.GOOS == "windows" { + data, err := exec.Command("wmic", "computersystem", "get", "model").Output() + if err != nil { + return false + } + fields := strings.Split(strings.TrimSpace(string(data)), "\r\n") + if len(fields) != 2 { + klog.V(2).Infof("Received unexpected value retrieving system model: %q", string(data)) + return false + } + name = fields[1] + } else { + data, err := ioutil.ReadFile(GCEProductNameFile) + if err != nil { + klog.V(2).Infof("Error while reading product_name: %v", err) + return false + } + name = strings.TrimSpace(string(data)) + } + return name == "Google" || name == "Google Compute Engine" +} + +// Enabled implements DockerConfigProvider for all of the Google implementations. +func (g *MetadataProvider) Enabled() bool { + return onGCEVM() +} + +// Provide implements DockerConfigProvider +func (g *DockerConfigKeyProvider) Provide(image string) credentialconfig.DockerConfig { + // Read the contents of the google-dockercfg metadata key and + // parse them as an alternate .dockercfg + if cfg, err := credentialconfig.ReadDockerConfigFileFromURL(DockerConfigKey, g.Client, metadataHeader); err != nil { + klog.Errorf("while reading 'google-dockercfg' metadata: %v", err) + } else { + return cfg + } + + return credentialconfig.DockerConfig{} +} + +// Provide implements DockerConfigProvider +func (g *DockerConfigURLKeyProvider) Provide(image string) credentialconfig.DockerConfig { + // Read the contents of the google-dockercfg-url key and load a .dockercfg from there + if url, err := credentialconfig.ReadURL(DockerConfigURLKey, g.Client, metadataHeader); err != nil { + klog.Errorf("while reading 'google-dockercfg-url' metadata: %v", err) + } else { + if strings.HasPrefix(string(url), "http") { + if cfg, err := credentialconfig.ReadDockerConfigFileFromURL(string(url), g.Client, nil); err != nil { + klog.Errorf("while reading 'google-dockercfg-url'-specified url: %s, %v", string(url), err) + } else { + return cfg + } + } else { + // TODO(mattmoor): support reading alternate scheme URLs (e.g. gs:// or s3://) + klog.Errorf("Unsupported URL scheme: %s", string(url)) + } + } + + return credentialconfig.DockerConfig{} +} + +// runWithBackoff runs input function `f` with an exponential backoff. +// Note that this method can block indefinitely. +func runWithBackoff(f func() ([]byte, error)) []byte { + var backoff = 100 * time.Millisecond + const maxBackoff = time.Minute + for { + value, err := f() + if err == nil { + return value + } + time.Sleep(backoff) + backoff = backoff * 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + } +} + +// Enabled implements a special metadata-based check, which verifies the +// storage scope is available on the GCE VM. +// If running on a GCE VM, check if 'default' service account exists. +// If it does not exist, assume that registry is not enabled. +// If default service account exists, check if relevant scopes exist in the default service account. +// The metadata service can become temporarily inaccesible. Hence all requests to the metadata +// service will be retried until the metadata server returns a `200`. +// It is expected that "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/" will return a `200` +// and "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/scopes" will also return `200`. +// More information on metadata service can be found here - https://cloud.google.com/compute/docs/storing-retrieving-metadata +func (g *ContainerRegistryProvider) Enabled() bool { + if !onGCEVM() { + return false + } + // Given that we are on GCE, we should keep retrying until the metadata server responds. + value := runWithBackoff(func() ([]byte, error) { + value, err := credentialconfig.ReadURL(serviceAccounts, g.Client, metadataHeader) + if err != nil { + klog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err) + } + return value, err + }) + // We expect the service account to return a list of account directories separated by newlines, e.g., + // sv-account-name1/ + // sv-account-name2/ + // ref: https://cloud.google.com/compute/docs/storing-retrieving-metadata + defaultServiceAccountExists := false + for _, sa := range strings.Split(string(value), "\n") { + if strings.TrimSpace(sa) == defaultServiceAccount { + defaultServiceAccountExists = true + break + } + } + if !defaultServiceAccountExists { + klog.V(2).Infof("'default' service account does not exist. Found following service accounts: %q", string(value)) + return false + } + url := metadataScopes + "?alt=json" + value = runWithBackoff(func() ([]byte, error) { + value, err := credentialconfig.ReadURL(url, g.Client, metadataHeader) + if err != nil { + klog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err) + } + return value, err + }) + var scopes []string + if err := json.Unmarshal(value, &scopes); err != nil { + klog.Errorf("Failed to unmarshal scopes: %v", err) + return false + } + for _, v := range scopes { + // cloudPlatformScope implies storage scope. + if strings.HasPrefix(v, StorageScopePrefix) || strings.HasPrefix(v, cloudPlatformScopePrefix) { + return true + } + } + klog.Warningf("Google container registry is disabled, no storage scope is available: %s", value) + return false +} + +// TokenBlob is used to decode the JSON blob containing an access token +// that is returned by GCE metadata. +type TokenBlob struct { + AccessToken string `json:"access_token"` +} + +// Provide implements DockerConfigProvider +func (g *ContainerRegistryProvider) Provide(image string) credentialconfig.DockerConfig { + cfg := credentialconfig.DockerConfig{} + + tokenJSONBlob, err := credentialconfig.ReadURL(metadataToken, g.Client, metadataHeader) + if err != nil { + klog.Errorf("while reading access token endpoint: %v", err) + return cfg + } + + email, err := credentialconfig.ReadURL(metadataEmail, g.Client, metadataHeader) + if err != nil { + klog.Errorf("while reading email endpoint: %v", err) + return cfg + } + + var parsedBlob TokenBlob + if err := json.Unmarshal([]byte(tokenJSONBlob), &parsedBlob); err != nil { + klog.Errorf("while parsing json blob %s: %v", tokenJSONBlob, err) + return cfg + } + + entry := credentialconfig.DockerConfigEntry{ + Username: "_token", + Password: parsedBlob.AccessToken, + Email: string(email), + } + + // Add our entry for each of the supported container registry URLs + for _, k := range containerRegistryUrls { + cfg[k] = entry + } + return cfg +} From acaea957ff7096ffd0d86cbb9e18a60697ea950a Mon Sep 17 00:00:00 2001 From: Kermit Alexander Date: Thu, 12 Nov 2020 02:16:45 +0000 Subject: [PATCH 2/3] Run ./hack/update-vendor.sh --- vendor/modules.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vendor/modules.txt b/vendor/modules.txt index a4fa1590d82..a05f59c107a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2181,6 +2181,7 @@ k8s.io/cloud-provider/controllers/route k8s.io/cloud-provider/controllers/service k8s.io/cloud-provider/controllers/service/config k8s.io/cloud-provider/controllers/service/config/v1alpha1 +k8s.io/cloud-provider/credentialconfig k8s.io/cloud-provider/fake k8s.io/cloud-provider/node/helpers k8s.io/cloud-provider/options @@ -2490,6 +2491,7 @@ k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/mockvmssvmclient k8s.io/legacy-cloud-providers/azure/metrics k8s.io/legacy-cloud-providers/azure/retry k8s.io/legacy-cloud-providers/gce +k8s.io/legacy-cloud-providers/gce/gcpcredential k8s.io/legacy-cloud-providers/openstack k8s.io/legacy-cloud-providers/vsphere k8s.io/legacy-cloud-providers/vsphere/testing From 0dcafb1f37ee522be3c045753623138e5b907001 Mon Sep 17 00:00:00 2001 From: Kermit Alexander Date: Thu, 12 Nov 2020 03:30:31 +0000 Subject: [PATCH 3/3] Add RegistryConfig/RegistryConfigEntry. --- cmd/kubelet/app/server.go | 5 +- pkg/credentialprovider/aws/aws_credentials.go | 21 +- .../azure/azure_credentials.go | 29 ++- .../credentialprovider}/config.go | 3 +- .../credentialprovider}/config_test.go | 2 +- pkg/credentialprovider/default_provider.go | 52 ---- pkg/credentialprovider/gcp/metadata.go | 209 +++++++++++++++- pkg/credentialprovider/gcp/metadata_test.go | 131 +++++----- pkg/credentialprovider/keyring.go | 8 +- pkg/credentialprovider/keyring_test.go | 41 +--- pkg/credentialprovider/plugins.go | 7 +- .../credentialprovider}/provider.go | 31 ++- .../credentialprovider}/provider_test.go | 19 +- pkg/credentialprovider/secrets/secrets.go | 9 +- .../credentialconfig/registry.go | 31 +++ .../gce/gcpcredential/credentialutil.go | 113 +++++++++ .../gce/gcpcredential/gcpcredential.go | 192 +++------------ .../gce/gcpcredential/registry_marshal.go | 110 +++++++++ .../gcpcredential/registry_marshal_test.go | 231 ++++++++++++++++++ 19 files changed, 875 insertions(+), 369 deletions(-) rename {staging/src/k8s.io/cloud-provider/credentialconfig => pkg/credentialprovider}/config.go (98%) rename {staging/src/k8s.io/cloud-provider/credentialconfig => pkg/credentialprovider}/config_test.go (99%) delete mode 100644 pkg/credentialprovider/default_provider.go rename {staging/src/k8s.io/cloud-provider/credentialconfig => pkg/credentialprovider}/provider.go (74%) rename {staging/src/k8s.io/cloud-provider/credentialconfig => pkg/credentialprovider}/provider_test.go (81%) create mode 100644 staging/src/k8s.io/cloud-provider/credentialconfig/registry.go create mode 100644 staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/credentialutil.go create mode 100644 staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/registry_marshal.go create mode 100644 staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/registry_marshal_test.go diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 41ec2181e0c..6a60dc3206d 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -59,8 +59,6 @@ import ( "k8s.io/client-go/util/connrotation" "k8s.io/client-go/util/keyutil" cloudprovider "k8s.io/cloud-provider" - "k8s.io/cloud-provider/credentialconfig" - "k8s.io/component-base/cli/flag" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/configz" "k8s.io/component-base/featuregate" @@ -74,6 +72,7 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/capabilities" + "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet" kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" @@ -1117,7 +1116,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie AllowPrivileged: true, }) - credentialconfig.SetPreferredDockercfgPath(kubeServer.RootDirectory) + credentialprovider.SetPreferredDockercfgPath(kubeServer.RootDirectory) klog.V(2).Infof("Using root directory: %v", kubeServer.RootDirectory) if kubeDeps.OSInterface == nil { diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go index 76bf7feee40..8f85237e8f3 100644 --- a/pkg/credentialprovider/aws/aws_credentials.go +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -33,7 +33,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" - "k8s.io/cloud-provider/credentialconfig" "k8s.io/component-base/version" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/credentialprovider" @@ -55,7 +54,7 @@ type ecrProvider struct { getterFactory tokenGetterFactory } -var _ credentialconfig.DockerConfigProvider = &ecrProvider{} +var _ credentialprovider.DockerConfigProvider = &ecrProvider{} func newECRProvider(getterFactory tokenGetterFactory) *ecrProvider { return &ecrProvider{ @@ -83,11 +82,11 @@ func (p *ecrProvider) Enabled() bool { // Provide returns a DockerConfig with credentials from the cache if they are // found, or from ECR -func (p *ecrProvider) Provide(image string) credentialconfig.DockerConfig { +func (p *ecrProvider) Provide(image string) credentialprovider.DockerConfig { parsed, err := parseRepoURL(image) if err != nil { klog.V(3).Info(err) - return credentialconfig.DockerConfig{} + return credentialprovider.DockerConfig{} } if cfg, exists := p.getFromCache(parsed); exists { @@ -99,15 +98,15 @@ func (p *ecrProvider) Provide(image string) credentialconfig.DockerConfig { cfg, err := p.getFromECR(parsed) if err != nil { klog.Errorf("error getting credentials from ECR for %s %v", parsed.registry, err) - return credentialconfig.DockerConfig{} + return credentialprovider.DockerConfig{} } klog.V(3).Infof("Got ECR credentials from ECR API for %s", parsed.registry) return cfg } // getFromCache attempts to get credentials from the cache -func (p *ecrProvider) getFromCache(parsed *parsedURL) (credentialconfig.DockerConfig, bool) { - cfg := credentialconfig.DockerConfig{} +func (p *ecrProvider) getFromCache(parsed *parsedURL) (credentialprovider.DockerConfig, bool) { + cfg := credentialprovider.DockerConfig{} obj, exists, err := p.cache.GetByKey(parsed.registry) if err != nil { @@ -125,8 +124,8 @@ func (p *ecrProvider) getFromCache(parsed *parsedURL) (credentialconfig.DockerCo } // getFromECR gets credentials from ECR since they are not in the cache -func (p *ecrProvider) getFromECR(parsed *parsedURL) (credentialconfig.DockerConfig, error) { - cfg := credentialconfig.DockerConfig{} +func (p *ecrProvider) getFromECR(parsed *parsedURL) (credentialprovider.DockerConfig, error) { + cfg := credentialprovider.DockerConfig{} getter, err := p.getterFactory.GetTokenGetterForRegion(parsed.region) if err != nil { return cfg, err @@ -261,7 +260,7 @@ func (p *ecrTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenI type cacheEntry struct { expiresAt time.Time - credentials credentialconfig.DockerConfigEntry + credentials credentialprovider.DockerConfigEntry registry string } @@ -276,7 +275,7 @@ func makeCacheEntry(data *ecr.AuthorizationData, registry string) (*cacheEntry, if len(parts) < 2 { return nil, errors.New("error getting username and password from authorization token") } - creds := credentialconfig.DockerConfigEntry{ + creds := credentialprovider.DockerConfigEntry{ Username: parts[0], Password: parts[1], Email: "not@val.id", // ECR doesn't care and Docker is about to obsolete it diff --git a/pkg/credentialprovider/azure/azure_credentials.go b/pkg/credentialprovider/azure/azure_credentials.go index 1fbd3705992..2d9281482bf 100644 --- a/pkg/credentialprovider/azure/azure_credentials.go +++ b/pkg/credentialprovider/azure/azure_credentials.go @@ -35,7 +35,6 @@ import ( "github.com/spf13/pflag" "k8s.io/client-go/tools/cache" - "k8s.io/cloud-provider/credentialconfig" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/legacy-cloud-providers/azure/auth" @@ -66,7 +65,7 @@ func init() { type cacheEntry struct { expiresAt time.Time - credentials credentialconfig.DockerConfigEntry + credentials credentialprovider.DockerConfigEntry registry string } @@ -123,7 +122,7 @@ func (az *azRegistriesClient) List(ctx context.Context) ([]containerregistry.Reg } // NewACRProvider parses the specified configFile and returns a DockerConfigProvider -func NewACRProvider(configFile *string) credentialconfig.DockerConfigProvider { +func NewACRProvider(configFile *string) credentialprovider.DockerConfigProvider { return &acrProvider{ file: configFile, cache: cache.NewExpirationStore(stringKeyFunc, &acrExpirationPolicy{}), @@ -208,8 +207,8 @@ func (a *acrProvider) Enabled() bool { } // getFromCache attempts to get credentials from the cache -func (a *acrProvider) getFromCache(loginServer string) (credentialconfig.DockerConfig, bool) { - cfg := credentialconfig.DockerConfig{} +func (a *acrProvider) getFromCache(loginServer string) (credentialprovider.DockerConfig, bool) { + cfg := credentialprovider.DockerConfig{} obj, exists, err := a.cache.GetByKey(loginServer) if err != nil { klog.Errorf("error getting ACR credentials from cache: %v", err) @@ -225,8 +224,8 @@ func (a *acrProvider) getFromCache(loginServer string) (credentialconfig.DockerC } // getFromACR gets credentials from ACR since they are not in the cache -func (a *acrProvider) getFromACR(loginServer string) (credentialconfig.DockerConfig, error) { - cfg := credentialconfig.DockerConfig{} +func (a *acrProvider) getFromACR(loginServer string) (credentialprovider.DockerConfig, error) { + cfg := credentialprovider.DockerConfig{} cred, err := getACRDockerEntryFromARMToken(a, loginServer) if err != nil { return cfg, err @@ -244,14 +243,14 @@ func (a *acrProvider) getFromACR(loginServer string) (credentialconfig.DockerCon return cfg, nil } -func (a *acrProvider) Provide(image string) credentialconfig.DockerConfig { +func (a *acrProvider) Provide(image string) credentialprovider.DockerConfig { loginServer := a.parseACRLoginServerFromImage(image) if loginServer == "" { klog.V(2).Infof("image(%s) is not from ACR, return empty authentication", image) - return credentialconfig.DockerConfig{} + return credentialprovider.DockerConfig{} } - cfg := credentialconfig.DockerConfig{} + cfg := credentialprovider.DockerConfig{} if a.config != nil && a.config.UseManagedIdentityExtension { var exists bool cfg, exists = a.getFromCache(loginServer) @@ -268,7 +267,7 @@ func (a *acrProvider) Provide(image string) credentialconfig.DockerConfig { } else { // Add our entry for each of the supported container registry URLs for _, url := range containerRegistryUrls { - cred := &credentialconfig.DockerConfigEntry{ + cred := &credentialprovider.DockerConfigEntry{ Username: a.config.AADClientID, Password: a.config.AADClientSecret, Email: dummyRegistryEmail, @@ -289,7 +288,7 @@ func (a *acrProvider) Provide(image string) credentialconfig.DockerConfig { } if !hasBeenAdded { - cred := &credentialconfig.DockerConfigEntry{ + cred := &credentialprovider.DockerConfigEntry{ Username: a.config.AADClientID, Password: a.config.AADClientSecret, Email: dummyRegistryEmail, @@ -300,7 +299,7 @@ func (a *acrProvider) Provide(image string) credentialconfig.DockerConfig { } // add ACR anonymous repo support: use empty username and password for anonymous access - defaultConfigEntry := credentialconfig.DockerConfigEntry{ + defaultConfigEntry := credentialprovider.DockerConfigEntry{ Username: "", Password: "", Email: dummyRegistryEmail, @@ -313,7 +312,7 @@ func getLoginServer(registry containerregistry.Registry) string { return *(*registry.RegistryProperties).LoginServer } -func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credentialconfig.DockerConfigEntry, error) { +func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credentialprovider.DockerConfigEntry, error) { // Run EnsureFresh to make sure the token is valid and does not expire if err := a.servicePrincipalToken.EnsureFresh(); err != nil { klog.Errorf("Failed to ensure fresh service principal token: %v", err) @@ -337,7 +336,7 @@ func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credent } klog.V(4).Infof("adding ACR docker config entry for: %s", loginServer) - return &credentialconfig.DockerConfigEntry{ + return &credentialprovider.DockerConfigEntry{ Username: dockerTokenLoginUsernameGUID, Password: registryRefreshToken, Email: dummyRegistryEmail, diff --git a/staging/src/k8s.io/cloud-provider/credentialconfig/config.go b/pkg/credentialprovider/config.go similarity index 98% rename from staging/src/k8s.io/cloud-provider/credentialconfig/config.go rename to pkg/credentialprovider/config.go index 838840a2a5f..00a5fa90608 100644 --- a/staging/src/k8s.io/cloud-provider/credentialconfig/config.go +++ b/pkg/credentialprovider/config.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package credentialconfig +package credentialprovider import ( "encoding/base64" @@ -232,6 +232,7 @@ func ReadDockerConfigFileFromURL(url string, client *http.Client, header *http.H return nil, err } +// ReadDockerConfigFileFromBytes read a docker config file from the given bytes func ReadDockerConfigFileFromBytes(contents []byte) (cfg DockerConfig, err error) { if err = json.Unmarshal(contents, &cfg); err != nil { return nil, errors.New("error occurred while trying to unmarshal json") diff --git a/staging/src/k8s.io/cloud-provider/credentialconfig/config_test.go b/pkg/credentialprovider/config_test.go similarity index 99% rename from staging/src/k8s.io/cloud-provider/credentialconfig/config_test.go rename to pkg/credentialprovider/config_test.go index 8a5aec0a52a..c9fe61c97e7 100644 --- a/staging/src/k8s.io/cloud-provider/credentialconfig/config_test.go +++ b/pkg/credentialprovider/config_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package credentialconfig +package credentialprovider import ( "encoding/base64" diff --git a/pkg/credentialprovider/default_provider.go b/pkg/credentialprovider/default_provider.go deleted file mode 100644 index f85ef49c83d..00000000000 --- a/pkg/credentialprovider/default_provider.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package credentialprovider - -import ( - "k8s.io/cloud-provider/credentialconfig" - "k8s.io/klog/v2" - "os" - "time" -) - -// A DockerConfigProvider that simply reads the .dockercfg file -type defaultDockerConfigProvider struct{} - -// init registers our default provider, which simply reads the .dockercfg file. -func init() { - RegisterCredentialProvider(".dockercfg", - &credentialconfig.CachingDockerConfigProvider{ - Provider: &defaultDockerConfigProvider{}, - Lifetime: 5 * time.Minute, - }) -} - -// Enabled implements dockerConfigProvider -func (d *defaultDockerConfigProvider) Enabled() bool { - return true -} - -// Provide implements dockerConfigProvider -func (d *defaultDockerConfigProvider) Provide(image string) credentialconfig.DockerConfig { - // Read the standard Docker credentials from .dockercfg - if cfg, err := credentialconfig.ReadDockerConfigFile(); err == nil { - return cfg - } else if !os.IsNotExist(err) { - klog.V(4).Infof("Unable to parse Docker config file: %v", err) - } - return credentialconfig.DockerConfig{} -} diff --git a/pkg/credentialprovider/gcp/metadata.go b/pkg/credentialprovider/gcp/metadata.go index a13cd4a42cb..9c5df1987c1 100644 --- a/pkg/credentialprovider/gcp/metadata.go +++ b/pkg/credentialprovider/gcp/metadata.go @@ -17,15 +17,46 @@ limitations under the License. package gcp import ( + "encoding/json" + "io/ioutil" "net/http" + "os/exec" + "runtime" + "strings" "time" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/cloud-provider/credentialconfig" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/legacy-cloud-providers/gce/gcpcredential" ) +const ( + metadataURL = "http://metadata.google.internal./computeMetadata/v1/" + metadataAttributes = metadataURL + "instance/attributes/" + // DockerConfigKey is the URL of the dockercfg metadata key used by DockerConfigKeyProvider. + DockerConfigKey = metadataAttributes + "google-dockercfg" + // DockerConfigURLKey is the URL of the dockercfg metadata key used by DockerConfigURLKeyProvider. + DockerConfigURLKey = metadataAttributes + "google-dockercfg-url" + serviceAccounts = metadataURL + "instance/service-accounts/" + metadataScopes = metadataURL + "instance/service-accounts/default/scopes" + metadataToken = metadataURL + "instance/service-accounts/default/token" + metadataEmail = metadataURL + "instance/service-accounts/default/email" + // StorageScopePrefix is the prefix checked by ContainerRegistryProvider.Enabled. + StorageScopePrefix = "https://www.googleapis.com/auth/devstorage" + cloudPlatformScopePrefix = "https://www.googleapis.com/auth/cloud-platform" + defaultServiceAccount = "default/" +) + +// gceProductNameFile is the product file path that contains the cloud service name. +// This is a variable instead of a const to enable testing. +var gceProductNameFile = "/sys/class/dmi/id/product_name" + +var metadataHeader = &http.Header{ + "Metadata-Flavor": []string{"Google"}, +} + // init registers the various means by which credentials may // be resolved on GCP. func init() { @@ -36,17 +67,17 @@ func init() { Timeout: metadataHTTPClientTimeout, } credentialprovider.RegisterCredentialProvider("google-dockercfg", - &credentialconfig.CachingDockerConfigProvider{ - Provider: &gcpcredential.DockerConfigKeyProvider{ - gcpcredential.MetadataProvider{Client: httpClient}, + &credentialprovider.CachingDockerConfigProvider{ + Provider: &DockerConfigKeyProvider{ + MetadataProvider: MetadataProvider{Client: httpClient}, }, Lifetime: 60 * time.Second, }) credentialprovider.RegisterCredentialProvider("google-dockercfg-url", - &credentialconfig.CachingDockerConfigProvider{ - Provider: &gcpcredential.DockerConfigURLKeyProvider{ - gcpcredential.MetadataProvider{Client: httpClient}, + &credentialprovider.CachingDockerConfigProvider{ + Provider: &DockerConfigURLKeyProvider{ + MetadataProvider: MetadataProvider{Client: httpClient}, }, Lifetime: 60 * time.Second, }) @@ -54,7 +85,169 @@ func init() { credentialprovider.RegisterCredentialProvider("google-container-registry", // Never cache this. The access token is already // cached by the metadata service. - &gcpcredential.ContainerRegistryProvider{ - gcpcredential.MetadataProvider{Client: httpClient}, + &ContainerRegistryProvider{ + MetadataProvider: MetadataProvider{Client: httpClient}, }) } + +// MetadataProvider is a DockerConfigProvider that reads its configuration from Google +// Compute Engine metadata. +type MetadataProvider struct { + Client *http.Client +} + +// DockerConfigKeyProvider is a DockerConfigProvider that reads its configuration from a specific +// Google Compute Engine metadata key: 'google-dockercfg'. +type DockerConfigKeyProvider struct { + MetadataProvider +} + +// DockerConfigURLKeyProvider is a DockerConfigProvider that reads its configuration from a URL read from +// a specific Google Compute Engine metadata key: 'google-dockercfg-url'. +type DockerConfigURLKeyProvider struct { + MetadataProvider +} + +// ContainerRegistryProvider is a DockerConfigProvider that provides a dockercfg with: +// Username: "_token" +// Password: "{access token from metadata}" +type ContainerRegistryProvider struct { + MetadataProvider +} + +// Returns true if it finds a local GCE VM. +// Looks at a product file that is an undocumented API. +func onGCEVM() bool { + var name string + + if runtime.GOOS == "windows" { + data, err := exec.Command("wmic", "computersystem", "get", "model").Output() + if err != nil { + return false + } + fields := strings.Split(strings.TrimSpace(string(data)), "\r\n") + if len(fields) != 2 { + klog.V(2).Infof("Received unexpected value retrieving system model: %q", string(data)) + return false + } + name = fields[1] + } else { + data, err := ioutil.ReadFile(gceProductNameFile) + if err != nil { + klog.V(2).Infof("Error while reading product_name: %v", err) + return false + } + name = strings.TrimSpace(string(data)) + } + return name == "Google" || name == "Google Compute Engine" +} + +// Enabled implements DockerConfigProvider for all of the Google implementations. +func (g *MetadataProvider) Enabled() bool { + return onGCEVM() +} + +// Provide implements DockerConfigProvider +func (g *DockerConfigKeyProvider) Provide(image string) credentialprovider.DockerConfig { + return registryToDocker(gcpcredential.ProvideConfigKey(g.Client, image)) +} + +// Provide implements DockerConfigProvider +func (g *DockerConfigURLKeyProvider) Provide(image string) credentialprovider.DockerConfig { + return registryToDocker(gcpcredential.ProvideURLKey(g.Client, image)) +} + +// runWithBackoff runs input function `f` with an exponential backoff. +// Note that this method can block indefinitely. +func runWithBackoff(f func() ([]byte, error)) []byte { + var backoff = 100 * time.Millisecond + const maxBackoff = time.Minute + for { + value, err := f() + if err == nil { + return value + } + time.Sleep(backoff) + backoff = backoff * 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + } +} + +// Enabled implements a special metadata-based check, which verifies the +// storage scope is available on the GCE VM. +// If running on a GCE VM, check if 'default' service account exists. +// If it does not exist, assume that registry is not enabled. +// If default service account exists, check if relevant scopes exist in the default service account. +// The metadata service can become temporarily inaccesible. Hence all requests to the metadata +// service will be retried until the metadata server returns a `200`. +// It is expected that "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/" will return a `200` +// and "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/scopes" will also return `200`. +// More information on metadata service can be found here - https://cloud.google.com/compute/docs/storing-retrieving-metadata +func (g *ContainerRegistryProvider) Enabled() bool { + if !onGCEVM() { + return false + } + // Given that we are on GCE, we should keep retrying until the metadata server responds. + value := runWithBackoff(func() ([]byte, error) { + value, err := gcpcredential.ReadURL(serviceAccounts, g.Client, metadataHeader) + if err != nil { + klog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err) + } + return value, err + }) + // We expect the service account to return a list of account directories separated by newlines, e.g., + // sv-account-name1/ + // sv-account-name2/ + // ref: https://cloud.google.com/compute/docs/storing-retrieving-metadata + defaultServiceAccountExists := false + for _, sa := range strings.Split(string(value), "\n") { + if strings.TrimSpace(sa) == defaultServiceAccount { + defaultServiceAccountExists = true + break + } + } + if !defaultServiceAccountExists { + klog.V(2).Infof("'default' service account does not exist. Found following service accounts: %q", string(value)) + return false + } + url := metadataScopes + "?alt=json" + value = runWithBackoff(func() ([]byte, error) { + value, err := gcpcredential.ReadURL(url, g.Client, metadataHeader) + if err != nil { + klog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err) + } + return value, err + }) + var scopes []string + if err := json.Unmarshal(value, &scopes); err != nil { + klog.Errorf("Failed to unmarshal scopes: %v", err) + return false + } + for _, v := range scopes { + // cloudPlatformScope implies storage scope. + if strings.HasPrefix(v, StorageScopePrefix) || strings.HasPrefix(v, cloudPlatformScopePrefix) { + return true + } + } + klog.Warningf("Google container registry is disabled, no storage scope is available: %s", value) + return false +} + +// Provide implements DockerConfigProvider +func (g *ContainerRegistryProvider) Provide(image string) credentialprovider.DockerConfig { + return registryToDocker(gcpcredential.ProvideContainerRegistry(g.Client, image)) +} + +func registryToDocker(registryConfig credentialconfig.RegistryConfig) credentialprovider.DockerConfig { + dockerConfig := credentialprovider.DockerConfig{} + for k, v := range registryConfig { + dockerConfig[k] = credentialprovider.DockerConfigEntry{ + Username: v.Username, + Password: v.Password, + Email: v.Email, + } + } + return dockerConfig +} diff --git a/pkg/credentialprovider/gcp/metadata_test.go b/pkg/credentialprovider/gcp/metadata_test.go index 5b4af49415a..30b1bb4204c 100644 --- a/pkg/credentialprovider/gcp/metadata_test.go +++ b/pkg/credentialprovider/gcp/metadata_test.go @@ -30,7 +30,6 @@ import ( "testing" utilnet "k8s.io/apimachinery/pkg/util/net" - "k8s.io/cloud-provider/credentialconfig" "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/legacy-cloud-providers/gce/gcpcredential" ) @@ -43,7 +42,40 @@ func createProductNameFile() (string, error) { return file.Name(), ioutil.WriteFile(file.Name(), []byte("Google"), 0600) } -func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) { +// The tests here are run in this fashion to ensure TestAllProvidersNoMetadata +// is run after the others, since that test currently relies upon the file +// referenced by gceProductNameFile being removed, which is the opposite of +// the other tests +func TestMetadata(t *testing.T) { + var err error + gceProductNameFile, err = createProductNameFile() + if err != nil { + t.Errorf("failed to create gce product name file: %v", err) + } + defer os.Remove(gceProductNameFile) + t.Run("productNameDependent", func(t *testing.T) { + t.Run("DockerKeyringFromGoogleDockerConfigMetadata", + DockerKeyringFromGoogleDockerConfigMetadata) + t.Run("DockerKeyringFromGoogleDockerConfigMetadataUrl", + DockerKeyringFromGoogleDockerConfigMetadataURL) + t.Run("ContainerRegistryNoServiceAccount", + ContainerRegistryNoServiceAccount) + t.Run("ContainerRegistryBasics", + ContainerRegistryBasics) + t.Run("ContainerRegistryNoStorageScope", + ContainerRegistryNoStorageScope) + t.Run("ComputePlatformScopeSubstitutesStorageScope", + ComputePlatformScopeSubstitutesStorageScope) + }) + // We defer os.Remove in case of an unexpected exit, but this os.Remove call + // is the normal teardown call so AllProvidersNoMetadata executes properly + os.Remove(gceProductNameFile) + t.Run("AllProvidersNoMetadata", + AllProvidersNoMetadata) +} + +func DockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) { + t.Parallel() registryURL := "hello.kubernetes.io" email := "foo@bar.baz" username := "foo" @@ -55,13 +87,6 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) { "auth": %q } }`, registryURL, email, auth) - - var err error - gcpcredential.GCEProductNameFile, err = createProductNameFile() - if err != nil { - t.Errorf("failed to create gce product name file: %v", err) - } - defer os.Remove(gcpcredential.GCEProductNameFile) const probeEndpoint = "/computeMetadata/v1/" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Only serve the one metadata key. @@ -85,8 +110,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) { }) keyring := &credentialprovider.BasicDockerKeyring{} - provider := &gcpcredential.DockerConfigKeyProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + provider := &DockerConfigKeyProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, } if !provider.Enabled() { @@ -116,7 +141,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) { } } -func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) { +func DockerKeyringFromGoogleDockerConfigMetadataURL(t *testing.T) { + t.Parallel() registryURL := "hello.kubernetes.io" email := "foo@bar.baz" username := "foo" @@ -128,13 +154,6 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) { "auth": %q } }`, registryURL, email, auth) - - var err error - gcpcredential.GCEProductNameFile, err = createProductNameFile() - if err != nil { - t.Errorf("failed to create gce product name file: %v", err) - } - defer os.Remove(gcpcredential.GCEProductNameFile) const probeEndpoint = "/computeMetadata/v1/" const valueEndpoint = "/my/value" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -163,8 +182,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) { }) keyring := &credentialprovider.BasicDockerKeyring{} - provider := &gcpcredential.DockerConfigURLKeyProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + provider := &DockerConfigURLKeyProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, } if !provider.Enabled() { @@ -194,7 +213,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) { } } -func TestContainerRegistryBasics(t *testing.T) { +func ContainerRegistryBasics(t *testing.T) { + t.Parallel() registryURLs := []string{"container.cloud.google.com", "eu.gcr.io", "us-west2-docker.pkg.dev"} for _, registryURL := range registryURLs { t.Run(registryURL, func(t *testing.T) { @@ -208,12 +228,6 @@ func TestContainerRegistryBasics(t *testing.T) { emailEndpoint = defaultEndpoint + "email" tokenEndpoint = defaultEndpoint + "token" ) - var err error - gcpcredential.GCEProductNameFile, err = createProductNameFile() - if err != nil { - t.Errorf("failed to create gce product name file: %v", err) - } - defer os.Remove(gcpcredential.GCEProductNameFile) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Only serve the URL key and the value endpoint @@ -249,8 +263,8 @@ func TestContainerRegistryBasics(t *testing.T) { }) keyring := &credentialprovider.BasicDockerKeyring{} - provider := &gcpcredential.ContainerRegistryProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + provider := &ContainerRegistryProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, } if !provider.Enabled() { @@ -282,7 +296,7 @@ func TestContainerRegistryBasics(t *testing.T) { } } -func TestContainerRegistryNoServiceAccount(t *testing.T) { +func ContainerRegistryNoServiceAccount(t *testing.T) { const ( serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/" ) @@ -302,13 +316,6 @@ func TestContainerRegistryNoServiceAccount(t *testing.T) { })) defer server.Close() - var err error - gcpcredential.GCEProductNameFile, err = createProductNameFile() - if err != nil { - t.Errorf("failed to create gce product name file: %v", err) - } - defer os.Remove(gcpcredential.GCEProductNameFile) - // Make a transport that reroutes all traffic to the example server transport := utilnet.SetTransportDefaults(&http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { @@ -316,8 +323,8 @@ func TestContainerRegistryNoServiceAccount(t *testing.T) { }, }) - provider := &gcpcredential.ContainerRegistryProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + provider := &ContainerRegistryProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, } if provider.Enabled() { @@ -325,7 +332,8 @@ func TestContainerRegistryNoServiceAccount(t *testing.T) { } } -func TestContainerRegistryNoStorageScope(t *testing.T) { +func ContainerRegistryNoStorageScope(t *testing.T) { + t.Parallel() const ( serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/" defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/" @@ -346,13 +354,6 @@ func TestContainerRegistryNoStorageScope(t *testing.T) { })) defer server.Close() - var err error - gcpcredential.GCEProductNameFile, err = createProductNameFile() - if err != nil { - t.Errorf("failed to create gce product name file: %v", err) - } - defer os.Remove(gcpcredential.GCEProductNameFile) - // Make a transport that reroutes all traffic to the example server transport := utilnet.SetTransportDefaults(&http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { @@ -360,8 +361,8 @@ func TestContainerRegistryNoStorageScope(t *testing.T) { }, }) - provider := &gcpcredential.ContainerRegistryProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + provider := &ContainerRegistryProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, } if provider.Enabled() { @@ -369,7 +370,8 @@ func TestContainerRegistryNoStorageScope(t *testing.T) { } } -func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) { +func ComputePlatformScopeSubstitutesStorageScope(t *testing.T) { + t.Parallel() const ( serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/" defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/" @@ -391,13 +393,6 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) { })) defer server.Close() - var err error - gcpcredential.GCEProductNameFile, err = createProductNameFile() - if err != nil { - t.Errorf("failed to create gce product name file: %v", err) - } - defer os.Remove(gcpcredential.GCEProductNameFile) - // Make a transport that reroutes all traffic to the example server transport := utilnet.SetTransportDefaults(&http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { @@ -405,8 +400,8 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) { }, }) - provider := &gcpcredential.ContainerRegistryProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + provider := &ContainerRegistryProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, } if !provider.Enabled() { @@ -414,7 +409,7 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) { } } -func TestAllProvidersNoMetadata(t *testing.T) { +func AllProvidersNoMetadata(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "", http.StatusNotFound) })) @@ -427,15 +422,15 @@ func TestAllProvidersNoMetadata(t *testing.T) { }, }) - providers := []credentialconfig.DockerConfigProvider{ - &gcpcredential.DockerConfigKeyProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + providers := []credentialprovider.DockerConfigProvider{ + &DockerConfigKeyProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, }, - &gcpcredential.DockerConfigURLKeyProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + &DockerConfigURLKeyProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, }, - &gcpcredential.ContainerRegistryProvider{ - gcpcredential.MetadataProvider{Client: &http.Client{Transport: transport}}, + &ContainerRegistryProvider{ + MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}}, }, } diff --git a/pkg/credentialprovider/keyring.go b/pkg/credentialprovider/keyring.go index 750bcdd26e8..0af960a384f 100644 --- a/pkg/credentialprovider/keyring.go +++ b/pkg/credentialprovider/keyring.go @@ -23,10 +23,8 @@ import ( "sort" "strings" - "k8s.io/klog/v2" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/cloud-provider/credentialconfig" + "k8s.io/klog/v2" ) // DockerKeyring tracks a set of docker registry credentials, maintaining a @@ -49,7 +47,7 @@ type BasicDockerKeyring struct { // providersDockerKeyring is an implementation of DockerKeyring that // materializes its dockercfg based on a set of dockerConfigProviders. type providersDockerKeyring struct { - Providers []credentialconfig.DockerConfigProvider + Providers []DockerConfigProvider } // AuthConfig contains authorization information for connecting to a Registry @@ -75,7 +73,7 @@ type AuthConfig struct { } // Add add some docker config in basic docker keyring -func (dk *BasicDockerKeyring) Add(cfg credentialconfig.DockerConfig) { +func (dk *BasicDockerKeyring) Add(cfg DockerConfig) { if dk.index == nil { dk.index = make([]string, 0) dk.creds = make(map[string][]AuthConfig) diff --git a/pkg/credentialprovider/keyring_test.go b/pkg/credentialprovider/keyring_test.go index dfbbf8a1275..07a63baa45a 100644 --- a/pkg/credentialprovider/keyring_test.go +++ b/pkg/credentialprovider/keyring_test.go @@ -21,8 +21,6 @@ import ( "fmt" "reflect" "testing" - - "k8s.io/cloud-provider/credentialconfig" ) func TestURLsMatch(t *testing.T) { @@ -205,7 +203,7 @@ func TestDockerKeyringForGlob(t *testing.T) { }`, test.globURL, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -273,7 +271,7 @@ func TestKeyringMiss(t *testing.T) { }`, test.globURL, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -301,7 +299,7 @@ func TestKeyringMissWithDockerHubCredentials(t *testing.T) { }`, url, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -327,7 +325,7 @@ func TestKeyringHitWithUnqualifiedDockerHub(t *testing.T) { }`, url, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -368,7 +366,7 @@ func TestKeyringHitWithUnqualifiedLibraryDockerHub(t *testing.T) { }`, url, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -409,7 +407,7 @@ func TestKeyringHitWithQualifiedDockerHub(t *testing.T) { }`, url, email, auth) keyring := &BasicDockerKeyring{} - if cfg, err := credentialconfig.ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { + if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { keyring.Add(cfg) @@ -456,27 +454,12 @@ func TestIsDefaultRegistryMatch(t *testing.T) { } } -type testProvider struct { - Count int -} - -// Enabled implements dockerConfigProvider -func (d *testProvider) Enabled() bool { - return true -} - -// Provide implements dockerConfigProvider -func (d *testProvider) Provide(image string) credentialconfig.DockerConfig { - d.Count++ - return credentialconfig.DockerConfig{} -} - func TestProvidersDockerKeyring(t *testing.T) { provider := &testProvider{ Count: 0, } keyring := &providersDockerKeyring{ - Providers: []credentialconfig.DockerConfigProvider{ + Providers: []DockerConfigProvider{ provider, }, } @@ -512,13 +495,13 @@ func TestDockerKeyringLookup(t *testing.T) { } dk := &BasicDockerKeyring{} - dk.Add(credentialconfig.DockerConfig{ - "bar.example.com/pong": credentialconfig.DockerConfigEntry{ + dk.Add(DockerConfig{ + "bar.example.com/pong": DockerConfigEntry{ Username: grace.Username, Password: grace.Password, Email: grace.Email, }, - "bar.example.com": credentialconfig.DockerConfigEntry{ + "bar.example.com": DockerConfigEntry{ Username: ada.Username, Password: ada.Password, Email: ada.Email, @@ -573,8 +556,8 @@ func TestIssue3797(t *testing.T) { } dk := &BasicDockerKeyring{} - dk.Add(credentialconfig.DockerConfig{ - "https://quay.io/v1/": credentialconfig.DockerConfigEntry{ + dk.Add(DockerConfig{ + "https://quay.io/v1/": DockerConfigEntry{ Username: rex.Username, Password: rex.Password, Email: rex.Email, diff --git a/pkg/credentialprovider/plugins.go b/pkg/credentialprovider/plugins.go index ed65aeec4c1..76051a9b4b5 100644 --- a/pkg/credentialprovider/plugins.go +++ b/pkg/credentialprovider/plugins.go @@ -21,20 +21,19 @@ import ( "sort" "sync" - "k8s.io/cloud-provider/credentialconfig" "k8s.io/klog/v2" ) // All registered credential providers. var providersMutex sync.Mutex -var providers = make(map[string]credentialconfig.DockerConfigProvider) +var providers = make(map[string]DockerConfigProvider) // RegisterCredentialProvider is called by provider implementations on // initialization to register themselves, like so: // func init() { // RegisterCredentialProvider("name", &myProvider{...}) // } -func RegisterCredentialProvider(name string, provider credentialconfig.DockerConfigProvider) { +func RegisterCredentialProvider(name string, provider DockerConfigProvider) { providersMutex.Lock() defer providersMutex.Unlock() _, found := providers[name] @@ -49,7 +48,7 @@ func RegisterCredentialProvider(name string, provider credentialconfig.DockerCon // which draws from the set of registered credential providers. func NewDockerKeyring() DockerKeyring { keyring := &providersDockerKeyring{ - Providers: make([]credentialconfig.DockerConfigProvider, 0), + Providers: make([]DockerConfigProvider, 0), } keys := reflect.ValueOf(providers).MapKeys() diff --git a/staging/src/k8s.io/cloud-provider/credentialconfig/provider.go b/pkg/credentialprovider/provider.go similarity index 74% rename from staging/src/k8s.io/cloud-provider/credentialconfig/provider.go rename to pkg/credentialprovider/provider.go index 37943fda833..8c9ad347b72 100644 --- a/staging/src/k8s.io/cloud-provider/credentialconfig/provider.go +++ b/pkg/credentialprovider/provider.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package credentialconfig +package credentialprovider import ( + "os" "reflect" "sync" "time" @@ -38,6 +39,18 @@ type DockerConfigProvider interface { Provide(image string) DockerConfig } +// A DockerConfigProvider that simply reads the .dockercfg file +type defaultDockerConfigProvider struct{} + +// init registers our default provider, which simply reads the .dockercfg file. +func init() { + RegisterCredentialProvider(".dockercfg", + &CachingDockerConfigProvider{ + Provider: &defaultDockerConfigProvider{}, + Lifetime: 5 * time.Minute, + }) +} + // CachingDockerConfigProvider implements DockerConfigProvider by composing // with another DockerConfigProvider and caching the DockerConfig it provides // for a pre-specified lifetime. @@ -55,6 +68,22 @@ type CachingDockerConfigProvider struct { mu sync.Mutex } +// Enabled implements dockerConfigProvider +func (d *defaultDockerConfigProvider) Enabled() bool { + return true +} + +// Provide implements dockerConfigProvider +func (d *defaultDockerConfigProvider) Provide(image string) DockerConfig { + // Read the standard Docker credentials from .dockercfg + if cfg, err := ReadDockerConfigFile(); err == nil { + return cfg + } else if !os.IsNotExist(err) { + klog.V(2).Infof("Docker config file not found: %v", err) + } + return DockerConfig{} +} + // Enabled implements dockerConfigProvider func (d *CachingDockerConfigProvider) Enabled() bool { return d.Provider.Enabled() diff --git a/staging/src/k8s.io/cloud-provider/credentialconfig/provider_test.go b/pkg/credentialprovider/provider_test.go similarity index 81% rename from staging/src/k8s.io/cloud-provider/credentialconfig/provider_test.go rename to pkg/credentialprovider/provider_test.go index 254af4bfc42..78ba24347b5 100644 --- a/staging/src/k8s.io/cloud-provider/credentialconfig/provider_test.go +++ b/pkg/credentialprovider/provider_test.go @@ -1,5 +1,5 @@ /* -Copyright 2014 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,13 +14,28 @@ See the License for the specific language governing permissions and limitations under the License. */ -package credentialconfig +package credentialprovider import ( "testing" "time" ) +type testProvider struct { + Count int +} + +// Enabled implements dockerConfigProvider +func (d *testProvider) Enabled() bool { + return true +} + +// Provide implements dockerConfigProvider +func (d *testProvider) Provide(image string) DockerConfig { + d.Count++ + return DockerConfig{} +} + func TestCachingProvider(t *testing.T) { provider := &testProvider{ Count: 0, diff --git a/pkg/credentialprovider/secrets/secrets.go b/pkg/credentialprovider/secrets/secrets.go index fb4664896ba..423cd2bbf94 100644 --- a/pkg/credentialprovider/secrets/secrets.go +++ b/pkg/credentialprovider/secrets/secrets.go @@ -19,8 +19,7 @@ package secrets import ( "encoding/json" - "k8s.io/api/core/v1" - "k8s.io/cloud-provider/credentialconfig" + v1 "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/credentialprovider" ) @@ -28,17 +27,17 @@ import ( // then a DockerKeyring is built based on every hit and unioned with the defaultKeyring. // If they do not, then the default keyring is returned func MakeDockerKeyring(passedSecrets []v1.Secret, defaultKeyring credentialprovider.DockerKeyring) (credentialprovider.DockerKeyring, error) { - passedCredentials := []credentialconfig.DockerConfig{} + passedCredentials := []credentialprovider.DockerConfig{} for _, passedSecret := range passedSecrets { if dockerConfigJSONBytes, dockerConfigJSONExists := passedSecret.Data[v1.DockerConfigJsonKey]; (passedSecret.Type == v1.SecretTypeDockerConfigJson) && dockerConfigJSONExists && (len(dockerConfigJSONBytes) > 0) { - dockerConfigJSON := credentialconfig.DockerConfigJSON{} + dockerConfigJSON := credentialprovider.DockerConfigJSON{} if err := json.Unmarshal(dockerConfigJSONBytes, &dockerConfigJSON); err != nil { return nil, err } passedCredentials = append(passedCredentials, dockerConfigJSON.Auths) } else if dockercfgBytes, dockercfgExists := passedSecret.Data[v1.DockerConfigKey]; (passedSecret.Type == v1.SecretTypeDockercfg) && dockercfgExists && (len(dockercfgBytes) > 0) { - dockercfg := credentialconfig.DockerConfig{} + dockercfg := credentialprovider.DockerConfig{} if err := json.Unmarshal(dockercfgBytes, &dockercfg); err != nil { return nil, err } diff --git a/staging/src/k8s.io/cloud-provider/credentialconfig/registry.go b/staging/src/k8s.io/cloud-provider/credentialconfig/registry.go new file mode 100644 index 00000000000..eac2173e8ad --- /dev/null +++ b/staging/src/k8s.io/cloud-provider/credentialconfig/registry.go @@ -0,0 +1,31 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package credentialconfig + +// Code taken from /pkg/credentialprovider/config.go + +// RegistryConfig represents the config file used by the docker CLI. +// This config that represents the credentials that should be used +// when pulling images from specific image repositories. +type RegistryConfig map[string]RegistryConfigEntry + +// RegistryConfigEntry wraps a docker config as a entry +type RegistryConfigEntry struct { + Username string + Password string + Email string +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/credentialutil.go b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/credentialutil.go new file mode 100644 index 00000000000..6f13eec0791 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/credentialutil.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcpcredential + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + + "k8s.io/cloud-provider/credentialconfig" + "k8s.io/klog/v2" +) + +const ( + maxReadLength = 10 * 1 << 20 // 10MB +) + +// HTTPError wraps a non-StatusOK error code as an error. +type HTTPError struct { + StatusCode int + URL string +} + +// Error implements error +func (he *HTTPError) Error() string { + return fmt.Sprintf("http status code: %d while fetching url %s", + he.StatusCode, he.URL) +} + +// ReadURL read contents from given url +func ReadURL(url string, client *http.Client, header *http.Header) (body []byte, err error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + if header != nil { + req.Header = *header + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + klog.V(2).Infof("body of failing http response: %v", resp.Body) + return nil, &HTTPError{ + StatusCode: resp.StatusCode, + URL: url, + } + } + + limitedReader := &io.LimitedReader{R: resp.Body, N: maxReadLength} + contents, err := ioutil.ReadAll(limitedReader) + if err != nil { + return nil, err + } + + if limitedReader.N <= 0 { + return nil, errors.New("the read limit is reached") + } + + return contents, nil +} + +// ReadDockerConfigFileFromURL read a docker config file from the given url +func ReadDockerConfigFileFromURL(url string, client *http.Client, header *http.Header) (cfg credentialconfig.RegistryConfig, err error) { + if contents, err := ReadURL(url, client, header); err == nil { + return ReadDockerConfigFileFromBytes(contents) + } + + return nil, err +} + +type internalRegistryConfig map[string]RegistryConfigEntry + +// ReadDockerConfigFileFromBytes read a docker config file from the given bytes +func ReadDockerConfigFileFromBytes(contents []byte) (cfg credentialconfig.RegistryConfig, err error) { + serializableCfg := internalRegistryConfig{} + if err = json.Unmarshal(contents, &serializableCfg); err != nil { + return nil, errors.New("error occurred while trying to unmarshal json") + } + return convertToExternalConfig(serializableCfg), nil +} + +func convertToExternalConfig(in internalRegistryConfig) (cfg credentialconfig.RegistryConfig) { + configMap := credentialconfig.RegistryConfig{} + for k, v := range in { + configMap[k] = credentialconfig.RegistryConfigEntry{ + Username: v.Username, + Password: v.Password, + Email: v.Email, + } + } + return configMap +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/gcpcredential.go b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/gcpcredential.go index 1105a339067..b51990d1543 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/gcpcredential.go +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/gcpcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2014 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,32 +18,31 @@ package gcpcredential import ( "encoding/json" - "io/ioutil" "net/http" - "os/exec" - "runtime" "strings" - "time" "k8s.io/cloud-provider/credentialconfig" "k8s.io/klog/v2" ) const ( - metadataURL = "http://metadata.google.internal./computeMetadata/v1/" - metadataAttributes = metadataURL + "instance/attributes/" - DockerConfigKey = metadataAttributes + "google-dockercfg" - DockerConfigURLKey = metadataAttributes + "google-dockercfg-url" - serviceAccounts = metadataURL + "instance/service-accounts/" - metadataScopes = metadataURL + "instance/service-accounts/default/scopes" - metadataToken = metadataURL + "instance/service-accounts/default/token" - metadataEmail = metadataURL + "instance/service-accounts/default/email" + metadataURL = "http://metadata.google.internal./computeMetadata/v1/" + metadataAttributes = metadataURL + "instance/attributes/" + // DockerConfigKey is the URL of the dockercfg metadata key used by DockerConfigKeyProvider. + DockerConfigKey = metadataAttributes + "google-dockercfg" + // DockerConfigURLKey is the URL of the dockercfg metadata key used by DockerConfigURLKeyProvider. + DockerConfigURLKey = metadataAttributes + "google-dockercfg-url" + serviceAccounts = metadataURL + "instance/service-accounts/" + metadataScopes = metadataURL + "instance/service-accounts/default/scopes" + metadataToken = metadataURL + "instance/service-accounts/default/token" + metadataEmail = metadataURL + "instance/service-accounts/default/email" + // StorageScopePrefix is the prefix checked by ContainerRegistryProvider.Enabled. StorageScopePrefix = "https://www.googleapis.com/auth/devstorage" cloudPlatformScopePrefix = "https://www.googleapis.com/auth/cloud-platform" defaultServiceAccount = "default/" ) -// Product file path that contains the cloud service name. +// GCEProductNameFile is the product file path that contains the cloud service name. // This is a variable instead of a const to enable testing. var GCEProductNameFile = "/sys/class/dmi/id/product_name" @@ -55,84 +54,27 @@ var metadataHeader = &http.Header{ "Metadata-Flavor": []string{"Google"}, } -// A DockerConfigProvider that reads its configuration from Google -// Compute Engine metadata. -type MetadataProvider struct { - Client *http.Client -} - -// A DockerConfigProvider that reads its configuration from a specific -// Google Compute Engine metadata key: 'google-dockercfg'. -type DockerConfigKeyProvider struct { - MetadataProvider -} - -// A DockerConfigProvider that reads its configuration from a URL read from -// a specific Google Compute Engine metadata key: 'google-dockercfg-url'. -type DockerConfigURLKeyProvider struct { - MetadataProvider -} - -// A DockerConfigProvider that provides a dockercfg with: -// Username: "_token" -// Password: "{access token from metadata}" -type ContainerRegistryProvider struct { - MetadataProvider -} - -// Returns true if it finds a local GCE VM. -// Looks at a product file that is an undocumented API. -func onGCEVM() bool { - var name string - - if runtime.GOOS == "windows" { - data, err := exec.Command("wmic", "computersystem", "get", "model").Output() - if err != nil { - return false - } - fields := strings.Split(strings.TrimSpace(string(data)), "\r\n") - if len(fields) != 2 { - klog.V(2).Infof("Received unexpected value retrieving system model: %q", string(data)) - return false - } - name = fields[1] - } else { - data, err := ioutil.ReadFile(GCEProductNameFile) - if err != nil { - klog.V(2).Infof("Error while reading product_name: %v", err) - return false - } - name = strings.TrimSpace(string(data)) - } - return name == "Google" || name == "Google Compute Engine" -} - -// Enabled implements DockerConfigProvider for all of the Google implementations. -func (g *MetadataProvider) Enabled() bool { - return onGCEVM() -} - -// Provide implements DockerConfigProvider -func (g *DockerConfigKeyProvider) Provide(image string) credentialconfig.DockerConfig { +// ProvideConfigKey implements a dockercfg-based authentication flow. +func ProvideConfigKey(client *http.Client, image string) credentialconfig.RegistryConfig { // Read the contents of the google-dockercfg metadata key and // parse them as an alternate .dockercfg - if cfg, err := credentialconfig.ReadDockerConfigFileFromURL(DockerConfigKey, g.Client, metadataHeader); err != nil { + if cfg, err := ReadDockerConfigFileFromURL(DockerConfigKey, client, metadataHeader); err != nil { klog.Errorf("while reading 'google-dockercfg' metadata: %v", err) } else { return cfg } - return credentialconfig.DockerConfig{} + return credentialconfig.RegistryConfig{} } -// Provide implements DockerConfigProvider -func (g *DockerConfigURLKeyProvider) Provide(image string) credentialconfig.DockerConfig { +// ProvideURLKey implements a dockercfg-url-based authentication flow. +func ProvideURLKey(client *http.Client, image string) credentialconfig.RegistryConfig { // Read the contents of the google-dockercfg-url key and load a .dockercfg from there - if url, err := credentialconfig.ReadURL(DockerConfigURLKey, g.Client, metadataHeader); err != nil { + if url, err := ReadURL(DockerConfigURLKey, client, metadataHeader); err != nil { klog.Errorf("while reading 'google-dockercfg-url' metadata: %v", err) } else { if strings.HasPrefix(string(url), "http") { - if cfg, err := credentialconfig.ReadDockerConfigFileFromURL(string(url), g.Client, nil); err != nil { + if cfg, err := ReadDockerConfigFileFromURL(string(url), client, nil); err != nil { klog.Errorf("while reading 'google-dockercfg-url'-specified url: %s, %v", string(url), err) } else { return cfg @@ -143,85 +85,7 @@ func (g *DockerConfigURLKeyProvider) Provide(image string) credentialconfig.Dock } } - return credentialconfig.DockerConfig{} -} - -// runWithBackoff runs input function `f` with an exponential backoff. -// Note that this method can block indefinitely. -func runWithBackoff(f func() ([]byte, error)) []byte { - var backoff = 100 * time.Millisecond - const maxBackoff = time.Minute - for { - value, err := f() - if err == nil { - return value - } - time.Sleep(backoff) - backoff = backoff * 2 - if backoff > maxBackoff { - backoff = maxBackoff - } - } -} - -// Enabled implements a special metadata-based check, which verifies the -// storage scope is available on the GCE VM. -// If running on a GCE VM, check if 'default' service account exists. -// If it does not exist, assume that registry is not enabled. -// If default service account exists, check if relevant scopes exist in the default service account. -// The metadata service can become temporarily inaccesible. Hence all requests to the metadata -// service will be retried until the metadata server returns a `200`. -// It is expected that "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/" will return a `200` -// and "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/scopes" will also return `200`. -// More information on metadata service can be found here - https://cloud.google.com/compute/docs/storing-retrieving-metadata -func (g *ContainerRegistryProvider) Enabled() bool { - if !onGCEVM() { - return false - } - // Given that we are on GCE, we should keep retrying until the metadata server responds. - value := runWithBackoff(func() ([]byte, error) { - value, err := credentialconfig.ReadURL(serviceAccounts, g.Client, metadataHeader) - if err != nil { - klog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err) - } - return value, err - }) - // We expect the service account to return a list of account directories separated by newlines, e.g., - // sv-account-name1/ - // sv-account-name2/ - // ref: https://cloud.google.com/compute/docs/storing-retrieving-metadata - defaultServiceAccountExists := false - for _, sa := range strings.Split(string(value), "\n") { - if strings.TrimSpace(sa) == defaultServiceAccount { - defaultServiceAccountExists = true - break - } - } - if !defaultServiceAccountExists { - klog.V(2).Infof("'default' service account does not exist. Found following service accounts: %q", string(value)) - return false - } - url := metadataScopes + "?alt=json" - value = runWithBackoff(func() ([]byte, error) { - value, err := credentialconfig.ReadURL(url, g.Client, metadataHeader) - if err != nil { - klog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err) - } - return value, err - }) - var scopes []string - if err := json.Unmarshal(value, &scopes); err != nil { - klog.Errorf("Failed to unmarshal scopes: %v", err) - return false - } - for _, v := range scopes { - // cloudPlatformScope implies storage scope. - if strings.HasPrefix(v, StorageScopePrefix) || strings.HasPrefix(v, cloudPlatformScopePrefix) { - return true - } - } - klog.Warningf("Google container registry is disabled, no storage scope is available: %s", value) - return false + return credentialconfig.RegistryConfig{} } // TokenBlob is used to decode the JSON blob containing an access token @@ -230,17 +94,17 @@ type TokenBlob struct { AccessToken string `json:"access_token"` } -// Provide implements DockerConfigProvider -func (g *ContainerRegistryProvider) Provide(image string) credentialconfig.DockerConfig { - cfg := credentialconfig.DockerConfig{} +// ProvideContainerRegistry implements a gcr.io-based authentication flow. +func ProvideContainerRegistry(client *http.Client, image string) credentialconfig.RegistryConfig { + cfg := credentialconfig.RegistryConfig{} - tokenJSONBlob, err := credentialconfig.ReadURL(metadataToken, g.Client, metadataHeader) + tokenJSONBlob, err := ReadURL(metadataToken, client, metadataHeader) if err != nil { klog.Errorf("while reading access token endpoint: %v", err) return cfg } - email, err := credentialconfig.ReadURL(metadataEmail, g.Client, metadataHeader) + email, err := ReadURL(metadataEmail, client, metadataHeader) if err != nil { klog.Errorf("while reading email endpoint: %v", err) return cfg @@ -252,7 +116,7 @@ func (g *ContainerRegistryProvider) Provide(image string) credentialconfig.Docke return cfg } - entry := credentialconfig.DockerConfigEntry{ + entry := credentialconfig.RegistryConfigEntry{ Username: "_token", Password: parsedBlob.AccessToken, Email: string(email), diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/registry_marshal.go b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/registry_marshal.go new file mode 100644 index 00000000000..75541e21166 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/registry_marshal.go @@ -0,0 +1,110 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcpcredential + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "k8s.io/cloud-provider/credentialconfig" +) + +// registryConfigEntryWithAuth is used solely for deserializing the Auth field +// into a dockerConfigEntry during JSON deserialization. +type registryConfigEntryWithAuth struct { + // +optional + Username string `json:"username,omitempty"` + // +optional + Password string `json:"password,omitempty"` + // +optional + Email string `json:"email,omitempty"` + // +optional + Auth string `json:"auth,omitempty"` +} + +// RegistryConfigEntry is a serializable wrapper around credentialconfig.RegistryConfigEntry. +type RegistryConfigEntry struct { + credentialconfig.RegistryConfigEntry +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (ident *RegistryConfigEntry) UnmarshalJSON(data []byte) error { + var tmp registryConfigEntryWithAuth + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + + ident.Username = tmp.Username + ident.Password = tmp.Password + ident.Email = tmp.Email + + if len(tmp.Auth) == 0 { + return nil + } + + ident.Username, ident.Password, err = decodeRegistryConfigFieldAuth(tmp.Auth) + return err +} + +// MarshalJSON implements the json.Marshaler interface. +func (ident RegistryConfigEntry) MarshalJSON() ([]byte, error) { + toEncode := registryConfigEntryWithAuth{ident.Username, ident.Password, ident.Email, ""} + toEncode.Auth = encodeRegistryConfigFieldAuth(ident.Username, ident.Password) + + return json.Marshal(toEncode) +} + +// decodeRegistryConfigFieldAuth deserializes the "auth" field from dockercfg into a +// username and a password. The format of the auth field is base64(:). +func decodeRegistryConfigFieldAuth(field string) (username, password string, err error) { + + var decoded []byte + + // StdEncoding can only decode padded string + // RawStdEncoding can only decode unpadded string + if strings.HasSuffix(strings.TrimSpace(field), "=") { + // decode padded data + decoded, err = base64.StdEncoding.DecodeString(field) + } else { + // decode unpadded data + decoded, err = base64.RawStdEncoding.DecodeString(field) + } + + if err != nil { + return + } + + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + err = fmt.Errorf("unable to parse auth field, must be formatted as base64(username:password)") + return + } + + username = parts[0] + password = parts[1] + + return +} + +func encodeRegistryConfigFieldAuth(username, password string) string { + fieldValue := username + ":" + password + + return base64.StdEncoding.EncodeToString([]byte(fieldValue)) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/registry_marshal_test.go b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/registry_marshal_test.go new file mode 100644 index 00000000000..0b5b145bd39 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/gcpcredential/registry_marshal_test.go @@ -0,0 +1,231 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcpcredential + +import ( + "encoding/base64" + "encoding/json" + "k8s.io/cloud-provider/credentialconfig" + "reflect" + "testing" +) + +// Code copied (and edited to replace DockerConfig* with RegistryConfig*) from: +// pkg/credentialprovider/config_test.go. + +func TestRegistryConfigEntryJSONDecode(t *testing.T) { + tests := []struct { + input []byte + expect RegistryConfigEntry + fail bool + }{ + // simple case, just decode the fields + { + // Fake values for testing. + input: []byte(`{"username": "foo", "password": "bar", "email": "foo@example.com"}`), + expect: RegistryConfigEntry{ + credentialconfig.RegistryConfigEntry{ + Username: "foo", + Password: "bar", + Email: "foo@example.com", + }, + }, + fail: false, + }, + + // auth field decodes to username & password + { + input: []byte(`{"auth": "Zm9vOmJhcg==", "email": "foo@example.com"}`), + expect: RegistryConfigEntry{ + credentialconfig.RegistryConfigEntry{ + Username: "foo", + Password: "bar", + Email: "foo@example.com", + }, + }, + fail: false, + }, + + // auth field overrides username & password + { + // Fake values for testing. + input: []byte(`{"username": "foo", "password": "bar", "auth": "cGluZzpwb25n", "email": "foo@example.com"}`), + expect: RegistryConfigEntry{ + credentialconfig.RegistryConfigEntry{ + Username: "ping", + Password: "pong", + Email: "foo@example.com", + }, + }, + fail: false, + }, + + // poorly-formatted auth causes failure + { + input: []byte(`{"auth": "pants", "email": "foo@example.com"}`), + expect: RegistryConfigEntry{ + credentialconfig.RegistryConfigEntry{ + Username: "", + Password: "", + Email: "foo@example.com", + }, + }, + fail: true, + }, + + // invalid JSON causes failure + { + input: []byte(`{"email": false}`), + expect: RegistryConfigEntry{ + credentialconfig.RegistryConfigEntry{ + Username: "", + Password: "", + Email: "", + }, + }, + fail: true, + }, + } + + for i, tt := range tests { + var output RegistryConfigEntry + err := json.Unmarshal(tt.input, &output) + if (err != nil) != tt.fail { + t.Errorf("case %d: expected fail=%t, got err=%v", i, tt.fail, err) + } + + if !reflect.DeepEqual(tt.expect, output) { + t.Errorf("case %d: expected output %#v, got %#v", i, tt.expect, output) + } + } +} + +func TestDecodeRegistryConfigFieldAuth(t *testing.T) { + tests := []struct { + input string + username string + password string + fail bool + }{ + // auth field decodes to username & password + { + input: "Zm9vOmJhcg==", + username: "foo", + password: "bar", + }, + + // some test as before but with field not well padded + { + input: "Zm9vOmJhcg", + username: "foo", + password: "bar", + }, + + // some test as before but with new line characters + { + input: "Zm9vOm\nJhcg==\n", + username: "foo", + password: "bar", + }, + + // standard encoding (with padding) + { + input: base64.StdEncoding.EncodeToString([]byte("foo:bar")), + username: "foo", + password: "bar", + }, + + // raw encoding (without padding) + { + input: base64.RawStdEncoding.EncodeToString([]byte("foo:bar")), + username: "foo", + password: "bar", + }, + + // the input is encoded with encodeRegistryConfigFieldAuth (standard encoding) + { + input: encodeRegistryConfigFieldAuth("foo", "bar"), + username: "foo", + password: "bar", + }, + + // good base64 data, but no colon separating username & password + { + input: "cGFudHM=", + fail: true, + }, + + // only new line characters are ignored + { + input: "Zm9vOmJhcg== ", + fail: true, + }, + + // bad base64 data + { + input: "pants", + fail: true, + }, + } + + for i, tt := range tests { + username, password, err := decodeRegistryConfigFieldAuth(tt.input) + if (err != nil) != tt.fail { + t.Errorf("case %d: expected fail=%t, got err=%v", i, tt.fail, err) + } + + if tt.username != username { + t.Errorf("case %d: expected username %q, got %q", i, tt.username, username) + } + + if tt.password != password { + t.Errorf("case %d: expected password %q, got %q", i, tt.password, password) + } + } +} + +func TestRegistryConfigEntryJSONCompatibleEncode(t *testing.T) { + tests := []struct { + input RegistryConfigEntry + expect []byte + }{ + // simple case, just decode the fields + { + // Fake values for testing. + expect: []byte(`{"username":"foo","password":"bar","email":"foo@example.com","auth":"Zm9vOmJhcg=="}`), + input: RegistryConfigEntry{ + credentialconfig.RegistryConfigEntry{ + Username: "foo", + Password: "bar", + Email: "foo@example.com", + }, + }, + }, + } + + for i, tt := range tests { + actual, err := json.Marshal(tt.input) + if err != nil { + t.Errorf("case %d: unexpected error: %v", i, err) + } + + if string(tt.expect) != string(actual) { + t.Errorf("case %d: expected %v, got %v", i, string(tt.expect), string(actual)) + } + } + +}