Merge pull request #95775 from DangerOnTheRanger/gcp-credential-staging

Move credential provider code to staging/
This commit is contained in:
Kubernetes Prow Robot 2021-03-08 22:53:00 -08:00 committed by GitHub
commit 45cf7e07be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 783 additions and 216 deletions

View File

@ -116,7 +116,7 @@ func ReadDockercfgFile(searchPaths []string) (cfg DockerConfig, err error) {
klog.V(4).Infof("while trying to read %s: %v", absDockerConfigFileLocation, err)
continue
}
cfg, err := readDockerConfigFileFromBytes(contents)
cfg, err := ReadDockerConfigFileFromBytes(contents)
if err != nil {
klog.V(4).Infof("couldn't get the config from %q contents: %v", absDockerConfigFileLocation, err)
continue
@ -226,13 +226,14 @@ func ReadURL(url string, client *http.Client, header *http.Header) (body []byte,
// ReadDockerConfigFileFromURL read a docker config file from the given url
func ReadDockerConfigFileFromURL(url string, client *http.Client, header *http.Header) (cfg DockerConfig, err error) {
if contents, err := ReadURL(url, client, header); err == nil {
return readDockerConfigFileFromBytes(contents)
return ReadDockerConfigFileFromBytes(contents)
}
return nil, err
}
func readDockerConfigFileFromBytes(contents []byte) (cfg DockerConfig, err error) {
// ReadDockerConfigFileFromBytes read a docker config file from the given bytes
func ReadDockerConfigFileFromBytes(contents []byte) (cfg DockerConfig, err error) {
if err = json.Unmarshal(contents, &cfg); err != nil {
return nil, errors.New("error occurred while trying to unmarshal json")
}

View File

@ -338,7 +338,7 @@ func TestReadDockerConfigFileFromBytes(t *testing.T) {
}
for _, tc := range testCases {
cfg, err := readDockerConfigFileFromBytes(tc.input)
cfg, err := ReadDockerConfigFileFromBytes(tc.input)
if err != nil && !tc.errorExpected {
t.Fatalf("Error was not expected: %v", err)
}

View File

@ -26,61 +26,37 @@ import (
"time"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/cloud-provider/credentialconfig"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/credentialprovider"
"k8s.io/legacy-cloud-providers/gce/gcpcredential"
)
const (
metadataURL = "http://metadata.google.internal./computeMetadata/v1/"
metadataAttributes = metadataURL + "instance/attributes/"
dockerConfigKey = metadataAttributes + "google-dockercfg"
dockerConfigURLKey = metadataAttributes + "google-dockercfg-url"
serviceAccounts = metadataURL + "instance/service-accounts/"
metadataScopes = metadataURL + "instance/service-accounts/default/scopes"
metadataToken = metadataURL + "instance/service-accounts/default/token"
metadataEmail = metadataURL + "instance/service-accounts/default/email"
storageScopePrefix = "https://www.googleapis.com/auth/devstorage"
metadataURL = "http://metadata.google.internal./computeMetadata/v1/"
metadataAttributes = metadataURL + "instance/attributes/"
// DockerConfigKey is the URL of the dockercfg metadata key used by DockerConfigKeyProvider.
DockerConfigKey = metadataAttributes + "google-dockercfg"
// DockerConfigURLKey is the URL of the dockercfg metadata key used by DockerConfigURLKeyProvider.
DockerConfigURLKey = metadataAttributes + "google-dockercfg-url"
serviceAccounts = metadataURL + "instance/service-accounts/"
metadataScopes = metadataURL + "instance/service-accounts/default/scopes"
metadataToken = metadataURL + "instance/service-accounts/default/token"
metadataEmail = metadataURL + "instance/service-accounts/default/email"
// StorageScopePrefix is the prefix checked by ContainerRegistryProvider.Enabled.
StorageScopePrefix = "https://www.googleapis.com/auth/devstorage"
cloudPlatformScopePrefix = "https://www.googleapis.com/auth/cloud-platform"
defaultServiceAccount = "default/"
)
// Product file path that contains the cloud service name.
// gceProductNameFile is the product file path that contains the cloud service name.
// This is a variable instead of a const to enable testing.
var gceProductNameFile = "/sys/class/dmi/id/product_name"
// For these urls, the parts of the host name can be glob, for example '*.gcr.io" will match
// "foo.gcr.io" and "bar.gcr.io".
var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io", "*.pkg.dev"}
var metadataHeader = &http.Header{
"Metadata-Flavor": []string{"Google"},
}
// A DockerConfigProvider that reads its configuration from Google
// Compute Engine metadata.
type metadataProvider struct {
Client *http.Client
}
// A DockerConfigProvider that reads its configuration from a specific
// Google Compute Engine metadata key: 'google-dockercfg'.
type dockerConfigKeyProvider struct {
metadataProvider
}
// A DockerConfigProvider that reads its configuration from a URL read from
// a specific Google Compute Engine metadata key: 'google-dockercfg-url'.
type dockerConfigURLKeyProvider struct {
metadataProvider
}
// A DockerConfigProvider that provides a dockercfg with:
// Username: "_token"
// Password: "{access token from metadata}"
type containerRegistryProvider struct {
metadataProvider
}
// init registers the various means by which credentials may
// be resolved on GCP.
func init() {
@ -92,16 +68,16 @@ func init() {
}
credentialprovider.RegisterCredentialProvider("google-dockercfg",
&credentialprovider.CachingDockerConfigProvider{
Provider: &dockerConfigKeyProvider{
metadataProvider{Client: httpClient},
Provider: &DockerConfigKeyProvider{
MetadataProvider: MetadataProvider{Client: httpClient},
},
Lifetime: 60 * time.Second,
})
credentialprovider.RegisterCredentialProvider("google-dockercfg-url",
&credentialprovider.CachingDockerConfigProvider{
Provider: &dockerConfigURLKeyProvider{
metadataProvider{Client: httpClient},
Provider: &DockerConfigURLKeyProvider{
MetadataProvider: MetadataProvider{Client: httpClient},
},
Lifetime: 60 * time.Second,
})
@ -109,11 +85,36 @@ func init() {
credentialprovider.RegisterCredentialProvider("google-container-registry",
// Never cache this. The access token is already
// cached by the metadata service.
&containerRegistryProvider{
metadataProvider{Client: httpClient},
&ContainerRegistryProvider{
MetadataProvider: MetadataProvider{Client: httpClient},
})
}
// MetadataProvider is a DockerConfigProvider that reads its configuration from Google
// Compute Engine metadata.
type MetadataProvider struct {
Client *http.Client
}
// DockerConfigKeyProvider is a DockerConfigProvider that reads its configuration from a specific
// Google Compute Engine metadata key: 'google-dockercfg'.
type DockerConfigKeyProvider struct {
MetadataProvider
}
// DockerConfigURLKeyProvider is a DockerConfigProvider that reads its configuration from a URL read from
// a specific Google Compute Engine metadata key: 'google-dockercfg-url'.
type DockerConfigURLKeyProvider struct {
MetadataProvider
}
// ContainerRegistryProvider is a DockerConfigProvider that provides a dockercfg with:
// Username: "_token"
// Password: "{access token from metadata}"
type ContainerRegistryProvider struct {
MetadataProvider
}
// Returns true if it finds a local GCE VM.
// Looks at a product file that is an undocumented API.
func onGCEVM() bool {
@ -142,42 +143,18 @@ func onGCEVM() bool {
}
// Enabled implements DockerConfigProvider for all of the Google implementations.
func (g *metadataProvider) Enabled() bool {
func (g *MetadataProvider) Enabled() bool {
return onGCEVM()
}
// Provide implements DockerConfigProvider
func (g *dockerConfigKeyProvider) Provide(image string) credentialprovider.DockerConfig {
// Read the contents of the google-dockercfg metadata key and
// parse them as an alternate .dockercfg
if cfg, err := credentialprovider.ReadDockerConfigFileFromURL(dockerConfigKey, g.Client, metadataHeader); err != nil {
klog.Errorf("while reading 'google-dockercfg' metadata: %v", err)
} else {
return cfg
}
return credentialprovider.DockerConfig{}
func (g *DockerConfigKeyProvider) Provide(image string) credentialprovider.DockerConfig {
return registryToDocker(gcpcredential.ProvideConfigKey(g.Client, image))
}
// Provide implements DockerConfigProvider
func (g *dockerConfigURLKeyProvider) Provide(image string) credentialprovider.DockerConfig {
// Read the contents of the google-dockercfg-url key and load a .dockercfg from there
if url, err := credentialprovider.ReadURL(dockerConfigURLKey, g.Client, metadataHeader); err != nil {
klog.Errorf("while reading 'google-dockercfg-url' metadata: %v", err)
} else {
if strings.HasPrefix(string(url), "http") {
if cfg, err := credentialprovider.ReadDockerConfigFileFromURL(string(url), g.Client, nil); err != nil {
klog.Errorf("while reading 'google-dockercfg-url'-specified url: %s, %v", string(url), err)
} else {
return cfg
}
} else {
// TODO(mattmoor): support reading alternate scheme URLs (e.g. gs:// or s3://)
klog.Errorf("Unsupported URL scheme: %s", string(url))
}
}
return credentialprovider.DockerConfig{}
func (g *DockerConfigURLKeyProvider) Provide(image string) credentialprovider.DockerConfig {
return registryToDocker(gcpcredential.ProvideURLKey(g.Client, image))
}
// runWithBackoff runs input function `f` with an exponential backoff.
@ -208,13 +185,13 @@ func runWithBackoff(f func() ([]byte, error)) []byte {
// It is expected that "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/" will return a `200`
// and "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/scopes" will also return `200`.
// More information on metadata service can be found here - https://cloud.google.com/compute/docs/storing-retrieving-metadata
func (g *containerRegistryProvider) Enabled() bool {
func (g *ContainerRegistryProvider) Enabled() bool {
if !onGCEVM() {
return false
}
// Given that we are on GCE, we should keep retrying until the metadata server responds.
value := runWithBackoff(func() ([]byte, error) {
value, err := credentialprovider.ReadURL(serviceAccounts, g.Client, metadataHeader)
value, err := gcpcredential.ReadURL(serviceAccounts, g.Client, metadataHeader)
if err != nil {
klog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err)
}
@ -237,7 +214,7 @@ func (g *containerRegistryProvider) Enabled() bool {
}
url := metadataScopes + "?alt=json"
value = runWithBackoff(func() ([]byte, error) {
value, err := credentialprovider.ReadURL(url, g.Client, metadataHeader)
value, err := gcpcredential.ReadURL(url, g.Client, metadataHeader)
if err != nil {
klog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err)
}
@ -250,7 +227,7 @@ func (g *containerRegistryProvider) Enabled() bool {
}
for _, v := range scopes {
// cloudPlatformScope implies storage scope.
if strings.HasPrefix(v, storageScopePrefix) || strings.HasPrefix(v, cloudPlatformScopePrefix) {
if strings.HasPrefix(v, StorageScopePrefix) || strings.HasPrefix(v, cloudPlatformScopePrefix) {
return true
}
}
@ -258,43 +235,19 @@ func (g *containerRegistryProvider) Enabled() bool {
return false
}
// tokenBlob is used to decode the JSON blob containing an access token
// that is returned by GCE metadata.
type tokenBlob struct {
AccessToken string `json:"access_token"`
}
// Provide implements DockerConfigProvider
func (g *containerRegistryProvider) Provide(image string) credentialprovider.DockerConfig {
cfg := credentialprovider.DockerConfig{}
tokenJSONBlob, err := credentialprovider.ReadURL(metadataToken, g.Client, metadataHeader)
if err != nil {
klog.Errorf("while reading access token endpoint: %v", err)
return cfg
}
email, err := credentialprovider.ReadURL(metadataEmail, g.Client, metadataHeader)
if err != nil {
klog.Errorf("while reading email endpoint: %v", err)
return cfg
}
var parsedBlob tokenBlob
if err := json.Unmarshal([]byte(tokenJSONBlob), &parsedBlob); err != nil {
klog.Errorf("while parsing json blob %s: %v", tokenJSONBlob, err)
return cfg
}
entry := credentialprovider.DockerConfigEntry{
Username: "_token",
Password: parsedBlob.AccessToken,
Email: string(email),
}
// Add our entry for each of the supported container registry URLs
for _, k := range containerRegistryUrls {
cfg[k] = entry
}
return cfg
func (g *ContainerRegistryProvider) Provide(image string) credentialprovider.DockerConfig {
return registryToDocker(gcpcredential.ProvideContainerRegistry(g.Client, image))
}
func registryToDocker(registryConfig credentialconfig.RegistryConfig) credentialprovider.DockerConfig {
dockerConfig := credentialprovider.DockerConfig{}
for k, v := range registryConfig {
dockerConfig[k] = credentialprovider.DockerConfigEntry{
Username: v.Username,
Password: v.Password,
Email: v.Email,
}
}
return dockerConfig
}

View File

@ -31,6 +31,7 @@ import (
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/kubernetes/pkg/credentialprovider"
"k8s.io/legacy-cloud-providers/gce/gcpcredential"
)
func createProductNameFile() (string, error) {
@ -41,7 +42,40 @@ func createProductNameFile() (string, error) {
return file.Name(), ioutil.WriteFile(file.Name(), []byte("Google"), 0600)
}
func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
// The tests here are run in this fashion to ensure TestAllProvidersNoMetadata
// is run after the others, since that test currently relies upon the file
// referenced by gceProductNameFile being removed, which is the opposite of
// the other tests
func TestMetadata(t *testing.T) {
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
t.Run("productNameDependent", func(t *testing.T) {
t.Run("DockerKeyringFromGoogleDockerConfigMetadata",
DockerKeyringFromGoogleDockerConfigMetadata)
t.Run("DockerKeyringFromGoogleDockerConfigMetadataUrl",
DockerKeyringFromGoogleDockerConfigMetadataURL)
t.Run("ContainerRegistryNoServiceAccount",
ContainerRegistryNoServiceAccount)
t.Run("ContainerRegistryBasics",
ContainerRegistryBasics)
t.Run("ContainerRegistryNoStorageScope",
ContainerRegistryNoStorageScope)
t.Run("ComputePlatformScopeSubstitutesStorageScope",
ComputePlatformScopeSubstitutesStorageScope)
})
// We defer os.Remove in case of an unexpected exit, but this os.Remove call
// is the normal teardown call so AllProvidersNoMetadata executes properly
os.Remove(gceProductNameFile)
t.Run("AllProvidersNoMetadata",
AllProvidersNoMetadata)
}
func DockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
t.Parallel()
registryURL := "hello.kubernetes.io"
email := "foo@bar.baz"
username := "foo"
@ -53,19 +87,12 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
"auth": %q
}
}`, registryURL, email, auth)
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
const probeEndpoint = "/computeMetadata/v1/"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the one metadata key.
if probeEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
} else if strings.HasSuffix(dockerConfigKey, r.URL.Path) {
} else if strings.HasSuffix(gcpcredential.DockerConfigKey, r.URL.Path) {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprintln(w, sampleDockerConfig)
@ -83,8 +110,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
})
keyring := &credentialprovider.BasicDockerKeyring{}
provider := &dockerConfigKeyProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
provider := &DockerConfigKeyProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
}
if !provider.Enabled() {
@ -114,7 +141,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
}
}
func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) {
func DockerKeyringFromGoogleDockerConfigMetadataURL(t *testing.T) {
t.Parallel()
registryURL := "hello.kubernetes.io"
email := "foo@bar.baz"
username := "foo"
@ -126,13 +154,6 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) {
"auth": %q
}
}`, registryURL, email, auth)
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
const probeEndpoint = "/computeMetadata/v1/"
const valueEndpoint = "/my/value"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -143,7 +164,7 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprintln(w, sampleDockerConfig)
} else if strings.HasSuffix(dockerConfigURLKey, r.URL.Path) {
} else if strings.HasSuffix(gcpcredential.DockerConfigURLKey, r.URL.Path) {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/text")
fmt.Fprint(w, "http://foo.bar.com"+valueEndpoint)
@ -161,8 +182,8 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) {
})
keyring := &credentialprovider.BasicDockerKeyring{}
provider := &dockerConfigURLKeyProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
provider := &DockerConfigURLKeyProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
}
if !provider.Enabled() {
@ -192,12 +213,13 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) {
}
}
func TestContainerRegistryBasics(t *testing.T) {
func ContainerRegistryBasics(t *testing.T) {
t.Parallel()
registryURLs := []string{"container.cloud.google.com", "eu.gcr.io", "us-west2-docker.pkg.dev"}
for _, registryURL := range registryURLs {
t.Run(registryURL, func(t *testing.T) {
email := "1234@project.gserviceaccount.com"
token := &tokenBlob{AccessToken: "ya26.lots-of-indiscernible-garbage"} // Fake value for testing.
token := &gcpcredential.TokenBlob{AccessToken: "ya26.lots-of-indiscernible-garbage"} // Fake value for testing.
const (
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
@ -206,19 +228,13 @@ func TestContainerRegistryBasics(t *testing.T) {
emailEndpoint = defaultEndpoint + "email"
tokenEndpoint = defaultEndpoint + "token"
)
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the URL key and the value endpoint
if scopeEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `["%s.read_write"]`, storageScopePrefix)
fmt.Fprintf(w, `["%s.read_write"]`, gcpcredential.StorageScopePrefix)
} else if emailEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, email)
@ -247,8 +263,8 @@ func TestContainerRegistryBasics(t *testing.T) {
})
keyring := &credentialprovider.BasicDockerKeyring{}
provider := &containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
provider := &ContainerRegistryProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
}
if !provider.Enabled() {
@ -280,7 +296,7 @@ func TestContainerRegistryBasics(t *testing.T) {
}
}
func TestContainerRegistryNoServiceAccount(t *testing.T) {
func ContainerRegistryNoServiceAccount(t *testing.T) {
const (
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
)
@ -300,13 +316,6 @@ func TestContainerRegistryNoServiceAccount(t *testing.T) {
}))
defer server.Close()
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
@ -314,8 +323,8 @@ func TestContainerRegistryNoServiceAccount(t *testing.T) {
},
})
provider := &containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
provider := &ContainerRegistryProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
}
if provider.Enabled() {
@ -323,7 +332,8 @@ func TestContainerRegistryNoServiceAccount(t *testing.T) {
}
}
func TestContainerRegistryNoStorageScope(t *testing.T) {
func ContainerRegistryNoStorageScope(t *testing.T) {
t.Parallel()
const (
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
@ -344,13 +354,6 @@ func TestContainerRegistryNoStorageScope(t *testing.T) {
}))
defer server.Close()
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
@ -358,8 +361,8 @@ func TestContainerRegistryNoStorageScope(t *testing.T) {
},
})
provider := &containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
provider := &ContainerRegistryProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
}
if provider.Enabled() {
@ -367,7 +370,8 @@ func TestContainerRegistryNoStorageScope(t *testing.T) {
}
}
func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
func ComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
t.Parallel()
const (
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
@ -389,13 +393,6 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
}))
defer server.Close()
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
@ -403,8 +400,8 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
},
})
provider := &containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
provider := &ContainerRegistryProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
}
if !provider.Enabled() {
@ -412,7 +409,7 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
}
}
func TestAllProvidersNoMetadata(t *testing.T) {
func AllProvidersNoMetadata(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "", http.StatusNotFound)
}))
@ -426,14 +423,14 @@ func TestAllProvidersNoMetadata(t *testing.T) {
})
providers := []credentialprovider.DockerConfigProvider{
&dockerConfigKeyProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
&DockerConfigKeyProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
},
&dockerConfigURLKeyProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
&DockerConfigURLKeyProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
},
&containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
&ContainerRegistryProvider{
MetadataProvider: MetadataProvider{Client: &http.Client{Transport: transport}},
},
}

View File

@ -23,9 +23,8 @@ import (
"sort"
"strings"
"k8s.io/klog/v2"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
)
// DockerKeyring tracks a set of docker registry credentials, maintaining a

View File

@ -203,7 +203,7 @@ func TestDockerKeyringForGlob(t *testing.T) {
}`, test.globURL, email, auth)
keyring := &BasicDockerKeyring{}
if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
} else {
keyring.Add(cfg)
@ -271,7 +271,7 @@ func TestKeyringMiss(t *testing.T) {
}`, test.globURL, email, auth)
keyring := &BasicDockerKeyring{}
if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
} else {
keyring.Add(cfg)
@ -299,7 +299,7 @@ func TestKeyringMissWithDockerHubCredentials(t *testing.T) {
}`, url, email, auth)
keyring := &BasicDockerKeyring{}
if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
} else {
keyring.Add(cfg)
@ -325,7 +325,7 @@ func TestKeyringHitWithUnqualifiedDockerHub(t *testing.T) {
}`, url, email, auth)
keyring := &BasicDockerKeyring{}
if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
} else {
keyring.Add(cfg)
@ -366,7 +366,7 @@ func TestKeyringHitWithUnqualifiedLibraryDockerHub(t *testing.T) {
}`, url, email, auth)
keyring := &BasicDockerKeyring{}
if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
} else {
keyring.Add(cfg)
@ -407,7 +407,7 @@ func TestKeyringHitWithQualifiedDockerHub(t *testing.T) {
}`, url, email, auth)
keyring := &BasicDockerKeyring{}
if cfg, err := readDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil {
t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err)
} else {
keyring.Add(cfg)
@ -454,21 +454,6 @@ func TestIsDefaultRegistryMatch(t *testing.T) {
}
}
type testProvider struct {
Count int
}
// Enabled implements dockerConfigProvider
func (d *testProvider) Enabled() bool {
return true
}
// Provide implements dockerConfigProvider
func (d *testProvider) Provide(image string) DockerConfig {
d.Count++
return DockerConfig{}
}
func TestProvidersDockerKeyring(t *testing.T) {
provider := &testProvider{
Count: 0,

View File

@ -79,7 +79,7 @@ func (d *defaultDockerConfigProvider) Provide(image string) DockerConfig {
if cfg, err := ReadDockerConfigFile(); err == nil {
return cfg
} else if !os.IsNotExist(err) {
klog.V(4).Infof("Unable to parse Docker config file: %v", err)
klog.V(2).Infof("Docker config file not found: %v", err)
}
return DockerConfig{}
}

View File

@ -1,5 +1,5 @@
/*
Copyright 2014 The Kubernetes Authors.
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -21,6 +21,21 @@ import (
"time"
)
type testProvider struct {
Count int
}
// Enabled implements dockerConfigProvider
func (d *testProvider) Enabled() bool {
return true
}
// Provide implements dockerConfigProvider
func (d *testProvider) Provide(image string) DockerConfig {
d.Count++
return DockerConfig{}
}
func TestCachingProvider(t *testing.T) {
provider := &testProvider{
Count: 0,

View File

@ -19,7 +19,7 @@ package secrets
import (
"encoding/json"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/credentialprovider"
)

View File

@ -0,0 +1,31 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package credentialconfig
// Code taken from /pkg/credentialprovider/config.go
// RegistryConfig represents the config file used by the docker CLI.
// This config that represents the credentials that should be used
// when pulling images from specific image repositories.
type RegistryConfig map[string]RegistryConfigEntry
// RegistryConfigEntry wraps a docker config as a entry
type RegistryConfigEntry struct {
Username string
Password string
Email string
}

View File

@ -0,0 +1,113 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gcpcredential
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"k8s.io/cloud-provider/credentialconfig"
"k8s.io/klog/v2"
)
const (
maxReadLength = 10 * 1 << 20 // 10MB
)
// HTTPError wraps a non-StatusOK error code as an error.
type HTTPError struct {
StatusCode int
URL string
}
// Error implements error
func (he *HTTPError) Error() string {
return fmt.Sprintf("http status code: %d while fetching url %s",
he.StatusCode, he.URL)
}
// ReadURL read contents from given url
func ReadURL(url string, client *http.Client, header *http.Header) (body []byte, err error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
if header != nil {
req.Header = *header
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
klog.V(2).Infof("body of failing http response: %v", resp.Body)
return nil, &HTTPError{
StatusCode: resp.StatusCode,
URL: url,
}
}
limitedReader := &io.LimitedReader{R: resp.Body, N: maxReadLength}
contents, err := ioutil.ReadAll(limitedReader)
if err != nil {
return nil, err
}
if limitedReader.N <= 0 {
return nil, errors.New("the read limit is reached")
}
return contents, nil
}
// ReadDockerConfigFileFromURL read a docker config file from the given url
func ReadDockerConfigFileFromURL(url string, client *http.Client, header *http.Header) (cfg credentialconfig.RegistryConfig, err error) {
if contents, err := ReadURL(url, client, header); err == nil {
return ReadDockerConfigFileFromBytes(contents)
}
return nil, err
}
type internalRegistryConfig map[string]RegistryConfigEntry
// ReadDockerConfigFileFromBytes read a docker config file from the given bytes
func ReadDockerConfigFileFromBytes(contents []byte) (cfg credentialconfig.RegistryConfig, err error) {
serializableCfg := internalRegistryConfig{}
if err = json.Unmarshal(contents, &serializableCfg); err != nil {
return nil, errors.New("error occurred while trying to unmarshal json")
}
return convertToExternalConfig(serializableCfg), nil
}
func convertToExternalConfig(in internalRegistryConfig) (cfg credentialconfig.RegistryConfig) {
configMap := credentialconfig.RegistryConfig{}
for k, v := range in {
configMap[k] = credentialconfig.RegistryConfigEntry{
Username: v.Username,
Password: v.Password,
Email: v.Email,
}
}
return configMap
}

View File

@ -0,0 +1,130 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gcpcredential
import (
"encoding/json"
"net/http"
"strings"
"k8s.io/cloud-provider/credentialconfig"
"k8s.io/klog/v2"
)
const (
metadataURL = "http://metadata.google.internal./computeMetadata/v1/"
metadataAttributes = metadataURL + "instance/attributes/"
// DockerConfigKey is the URL of the dockercfg metadata key used by DockerConfigKeyProvider.
DockerConfigKey = metadataAttributes + "google-dockercfg"
// DockerConfigURLKey is the URL of the dockercfg metadata key used by DockerConfigURLKeyProvider.
DockerConfigURLKey = metadataAttributes + "google-dockercfg-url"
serviceAccounts = metadataURL + "instance/service-accounts/"
metadataScopes = metadataURL + "instance/service-accounts/default/scopes"
metadataToken = metadataURL + "instance/service-accounts/default/token"
metadataEmail = metadataURL + "instance/service-accounts/default/email"
// StorageScopePrefix is the prefix checked by ContainerRegistryProvider.Enabled.
StorageScopePrefix = "https://www.googleapis.com/auth/devstorage"
cloudPlatformScopePrefix = "https://www.googleapis.com/auth/cloud-platform"
defaultServiceAccount = "default/"
)
// GCEProductNameFile is the product file path that contains the cloud service name.
// This is a variable instead of a const to enable testing.
var GCEProductNameFile = "/sys/class/dmi/id/product_name"
// For these urls, the parts of the host name can be glob, for example '*.gcr.io" will match
// "foo.gcr.io" and "bar.gcr.io".
var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io", "*.pkg.dev"}
var metadataHeader = &http.Header{
"Metadata-Flavor": []string{"Google"},
}
// ProvideConfigKey implements a dockercfg-based authentication flow.
func ProvideConfigKey(client *http.Client, image string) credentialconfig.RegistryConfig {
// Read the contents of the google-dockercfg metadata key and
// parse them as an alternate .dockercfg
if cfg, err := ReadDockerConfigFileFromURL(DockerConfigKey, client, metadataHeader); err != nil {
klog.Errorf("while reading 'google-dockercfg' metadata: %v", err)
} else {
return cfg
}
return credentialconfig.RegistryConfig{}
}
// ProvideURLKey implements a dockercfg-url-based authentication flow.
func ProvideURLKey(client *http.Client, image string) credentialconfig.RegistryConfig {
// Read the contents of the google-dockercfg-url key and load a .dockercfg from there
if url, err := ReadURL(DockerConfigURLKey, client, metadataHeader); err != nil {
klog.Errorf("while reading 'google-dockercfg-url' metadata: %v", err)
} else {
if strings.HasPrefix(string(url), "http") {
if cfg, err := ReadDockerConfigFileFromURL(string(url), client, nil); err != nil {
klog.Errorf("while reading 'google-dockercfg-url'-specified url: %s, %v", string(url), err)
} else {
return cfg
}
} else {
// TODO(mattmoor): support reading alternate scheme URLs (e.g. gs:// or s3://)
klog.Errorf("Unsupported URL scheme: %s", string(url))
}
}
return credentialconfig.RegistryConfig{}
}
// TokenBlob is used to decode the JSON blob containing an access token
// that is returned by GCE metadata.
type TokenBlob struct {
AccessToken string `json:"access_token"`
}
// ProvideContainerRegistry implements a gcr.io-based authentication flow.
func ProvideContainerRegistry(client *http.Client, image string) credentialconfig.RegistryConfig {
cfg := credentialconfig.RegistryConfig{}
tokenJSONBlob, err := ReadURL(metadataToken, client, metadataHeader)
if err != nil {
klog.Errorf("while reading access token endpoint: %v", err)
return cfg
}
email, err := ReadURL(metadataEmail, client, metadataHeader)
if err != nil {
klog.Errorf("while reading email endpoint: %v", err)
return cfg
}
var parsedBlob TokenBlob
if err := json.Unmarshal([]byte(tokenJSONBlob), &parsedBlob); err != nil {
klog.Errorf("while parsing json blob %s: %v", tokenJSONBlob, err)
return cfg
}
entry := credentialconfig.RegistryConfigEntry{
Username: "_token",
Password: parsedBlob.AccessToken,
Email: string(email),
}
// Add our entry for each of the supported container registry URLs
for _, k := range containerRegistryUrls {
cfg[k] = entry
}
return cfg
}

View File

@ -0,0 +1,110 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gcpcredential
import (
"encoding/base64"
"encoding/json"
"fmt"
"strings"
"k8s.io/cloud-provider/credentialconfig"
)
// registryConfigEntryWithAuth is used solely for deserializing the Auth field
// into a dockerConfigEntry during JSON deserialization.
type registryConfigEntryWithAuth struct {
// +optional
Username string `json:"username,omitempty"`
// +optional
Password string `json:"password,omitempty"`
// +optional
Email string `json:"email,omitempty"`
// +optional
Auth string `json:"auth,omitempty"`
}
// RegistryConfigEntry is a serializable wrapper around credentialconfig.RegistryConfigEntry.
type RegistryConfigEntry struct {
credentialconfig.RegistryConfigEntry
}
// UnmarshalJSON implements the json.Unmarshaler interface.
func (ident *RegistryConfigEntry) UnmarshalJSON(data []byte) error {
var tmp registryConfigEntryWithAuth
err := json.Unmarshal(data, &tmp)
if err != nil {
return err
}
ident.Username = tmp.Username
ident.Password = tmp.Password
ident.Email = tmp.Email
if len(tmp.Auth) == 0 {
return nil
}
ident.Username, ident.Password, err = decodeRegistryConfigFieldAuth(tmp.Auth)
return err
}
// MarshalJSON implements the json.Marshaler interface.
func (ident RegistryConfigEntry) MarshalJSON() ([]byte, error) {
toEncode := registryConfigEntryWithAuth{ident.Username, ident.Password, ident.Email, ""}
toEncode.Auth = encodeRegistryConfigFieldAuth(ident.Username, ident.Password)
return json.Marshal(toEncode)
}
// decodeRegistryConfigFieldAuth deserializes the "auth" field from dockercfg into a
// username and a password. The format of the auth field is base64(<username>:<password>).
func decodeRegistryConfigFieldAuth(field string) (username, password string, err error) {
var decoded []byte
// StdEncoding can only decode padded string
// RawStdEncoding can only decode unpadded string
if strings.HasSuffix(strings.TrimSpace(field), "=") {
// decode padded data
decoded, err = base64.StdEncoding.DecodeString(field)
} else {
// decode unpadded data
decoded, err = base64.RawStdEncoding.DecodeString(field)
}
if err != nil {
return
}
parts := strings.SplitN(string(decoded), ":", 2)
if len(parts) != 2 {
err = fmt.Errorf("unable to parse auth field, must be formatted as base64(username:password)")
return
}
username = parts[0]
password = parts[1]
return
}
func encodeRegistryConfigFieldAuth(username, password string) string {
fieldValue := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(fieldValue))
}

View File

@ -0,0 +1,231 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gcpcredential
import (
"encoding/base64"
"encoding/json"
"k8s.io/cloud-provider/credentialconfig"
"reflect"
"testing"
)
// Code copied (and edited to replace DockerConfig* with RegistryConfig*) from:
// pkg/credentialprovider/config_test.go.
func TestRegistryConfigEntryJSONDecode(t *testing.T) {
tests := []struct {
input []byte
expect RegistryConfigEntry
fail bool
}{
// simple case, just decode the fields
{
// Fake values for testing.
input: []byte(`{"username": "foo", "password": "bar", "email": "foo@example.com"}`),
expect: RegistryConfigEntry{
credentialconfig.RegistryConfigEntry{
Username: "foo",
Password: "bar",
Email: "foo@example.com",
},
},
fail: false,
},
// auth field decodes to username & password
{
input: []byte(`{"auth": "Zm9vOmJhcg==", "email": "foo@example.com"}`),
expect: RegistryConfigEntry{
credentialconfig.RegistryConfigEntry{
Username: "foo",
Password: "bar",
Email: "foo@example.com",
},
},
fail: false,
},
// auth field overrides username & password
{
// Fake values for testing.
input: []byte(`{"username": "foo", "password": "bar", "auth": "cGluZzpwb25n", "email": "foo@example.com"}`),
expect: RegistryConfigEntry{
credentialconfig.RegistryConfigEntry{
Username: "ping",
Password: "pong",
Email: "foo@example.com",
},
},
fail: false,
},
// poorly-formatted auth causes failure
{
input: []byte(`{"auth": "pants", "email": "foo@example.com"}`),
expect: RegistryConfigEntry{
credentialconfig.RegistryConfigEntry{
Username: "",
Password: "",
Email: "foo@example.com",
},
},
fail: true,
},
// invalid JSON causes failure
{
input: []byte(`{"email": false}`),
expect: RegistryConfigEntry{
credentialconfig.RegistryConfigEntry{
Username: "",
Password: "",
Email: "",
},
},
fail: true,
},
}
for i, tt := range tests {
var output RegistryConfigEntry
err := json.Unmarshal(tt.input, &output)
if (err != nil) != tt.fail {
t.Errorf("case %d: expected fail=%t, got err=%v", i, tt.fail, err)
}
if !reflect.DeepEqual(tt.expect, output) {
t.Errorf("case %d: expected output %#v, got %#v", i, tt.expect, output)
}
}
}
func TestDecodeRegistryConfigFieldAuth(t *testing.T) {
tests := []struct {
input string
username string
password string
fail bool
}{
// auth field decodes to username & password
{
input: "Zm9vOmJhcg==",
username: "foo",
password: "bar",
},
// some test as before but with field not well padded
{
input: "Zm9vOmJhcg",
username: "foo",
password: "bar",
},
// some test as before but with new line characters
{
input: "Zm9vOm\nJhcg==\n",
username: "foo",
password: "bar",
},
// standard encoding (with padding)
{
input: base64.StdEncoding.EncodeToString([]byte("foo:bar")),
username: "foo",
password: "bar",
},
// raw encoding (without padding)
{
input: base64.RawStdEncoding.EncodeToString([]byte("foo:bar")),
username: "foo",
password: "bar",
},
// the input is encoded with encodeRegistryConfigFieldAuth (standard encoding)
{
input: encodeRegistryConfigFieldAuth("foo", "bar"),
username: "foo",
password: "bar",
},
// good base64 data, but no colon separating username & password
{
input: "cGFudHM=",
fail: true,
},
// only new line characters are ignored
{
input: "Zm9vOmJhcg== ",
fail: true,
},
// bad base64 data
{
input: "pants",
fail: true,
},
}
for i, tt := range tests {
username, password, err := decodeRegistryConfigFieldAuth(tt.input)
if (err != nil) != tt.fail {
t.Errorf("case %d: expected fail=%t, got err=%v", i, tt.fail, err)
}
if tt.username != username {
t.Errorf("case %d: expected username %q, got %q", i, tt.username, username)
}
if tt.password != password {
t.Errorf("case %d: expected password %q, got %q", i, tt.password, password)
}
}
}
func TestRegistryConfigEntryJSONCompatibleEncode(t *testing.T) {
tests := []struct {
input RegistryConfigEntry
expect []byte
}{
// simple case, just decode the fields
{
// Fake values for testing.
expect: []byte(`{"username":"foo","password":"bar","email":"foo@example.com","auth":"Zm9vOmJhcg=="}`),
input: RegistryConfigEntry{
credentialconfig.RegistryConfigEntry{
Username: "foo",
Password: "bar",
Email: "foo@example.com",
},
},
},
}
for i, tt := range tests {
actual, err := json.Marshal(tt.input)
if err != nil {
t.Errorf("case %d: unexpected error: %v", i, err)
}
if string(tt.expect) != string(actual) {
t.Errorf("case %d: expected %v, got %v", i, string(tt.expect), string(actual))
}
}
}

2
vendor/modules.txt vendored
View File

@ -2207,6 +2207,7 @@ k8s.io/cloud-provider/controllers/route
k8s.io/cloud-provider/controllers/service
k8s.io/cloud-provider/controllers/service/config
k8s.io/cloud-provider/controllers/service/config/v1alpha1
k8s.io/cloud-provider/credentialconfig
k8s.io/cloud-provider/fake
k8s.io/cloud-provider/node/helpers
k8s.io/cloud-provider/options
@ -2522,6 +2523,7 @@ k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/mockvmssvmclient
k8s.io/legacy-cloud-providers/azure/metrics
k8s.io/legacy-cloud-providers/azure/retry
k8s.io/legacy-cloud-providers/gce
k8s.io/legacy-cloud-providers/gce/gcpcredential
k8s.io/legacy-cloud-providers/openstack
k8s.io/legacy-cloud-providers/vsphere
k8s.io/legacy-cloud-providers/vsphere/testing