Merge pull request #28871 from vishh/gce-cp

Automatic merge from submit-queue

Do not query the metadata server to find out if running on GCE.  Retry metadata server query for gcr if running on gce.

Retry the logic for determining is gcr is enabled to workaround metadata unavailability.

Note: This patch does not retry fetching registry credentials.

pkg/credentialprovider/gcp/metadata.go

@@ -18,6 +18,7 @@ package gcp_credentials
 
 import (
 	"encoding/json"
+	"io/ioutil"
 	"net/http"
 	"strings"
 	"time"
@@ -31,13 +32,20 @@ const (
 	metadataAttributes       = metadataUrl + "instance/attributes/"
 	dockerConfigKey          = metadataAttributes + "google-dockercfg"
 	dockerConfigUrlKey       = metadataAttributes + "google-dockercfg-url"
+	serviceAccounts          = metadataUrl + "instance/service-accounts/"
 	metadataScopes           = metadataUrl + "instance/service-accounts/default/scopes"
 	metadataToken            = metadataUrl + "instance/service-accounts/default/token"
 	metadataEmail            = metadataUrl + "instance/service-accounts/default/email"
 	storageScopePrefix       = "https://www.googleapis.com/auth/devstorage"
 	cloudPlatformScopePrefix = "https://www.googleapis.com/auth/cloud-platform"
+	googleProductName        = "Google"
+	defaultServiceAccount    = "default/"
 )
 
+// Product file path that contains the cloud service name.
+// This is a variable instead of a const to enable testing.
+var gceProductNameFile = "/sys/class/dmi/id/product_name"
+
 // For these urls, the parts of the host name can be glob, for example '*.gcr.io" will match
 // "foo.gcr.io" and "bar.gcr.io".
 var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io"}
@@ -98,10 +106,20 @@ func init() {
 		})
 }
 
+// Returns true if it finds a local GCE VM.
+// Looks at a product file that is an undocumented API.
+func onGCEVM() bool {
+	data, err := ioutil.ReadFile(gceProductNameFile)
+	if err != nil {
+		glog.V(2).Infof("Error while reading product_name: %v", err)
+		return false
+	}
+	return strings.Contains(string(data), googleProductName)
+}
+
 // Enabled implements DockerConfigProvider for all of the Google implementations.
 func (g *metadataProvider) Enabled() bool {
-	_, err := credentialprovider.ReadUrl(metadataUrl, g.Client, metadataHeader)
-	return err == nil
+	return onGCEVM()
 }
 
 // LazyProvide implements DockerConfigProvider. Should never be called.
@@ -148,18 +166,74 @@ func (g *dockerConfigUrlKeyProvider) Provide() credentialprovider.DockerConfig {
 	return credentialprovider.DockerConfig{}
 }
 
+// runcWithBackoff runs input function `f` with an exponential backoff.
+// Note that this method can block indefinitely.
+func runWithBackoff(f func() ([]byte, error)) []byte {
+	var backoff = 100 * time.Millisecond
+	const maxBackoff = time.Minute
+	for {
+		value, err := f()
+		if err == nil {
+			return value
+		}
+		time.Sleep(backoff)
+		backoff = backoff * 2
+		if backoff > maxBackoff {
+			backoff = maxBackoff
+		}
+	}
+}
+
 // Enabled implements a special metadata-based check, which verifies the
 // storage scope is available on the GCE VM.
+// If running on a GCE VM, check if 'default' service account exists.
+// If it does not exist, assume that registry is not enabled.
+// If default service account exists, check if relevant scopes exist in the default service account.
+// The metadata service can become temporarily inaccesible. Hence all requests to the metadata
+// service will be retried until the metadata server returns a `200`.
+// It is expected that "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/" will return a `200`
+// and "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/scopes" will also return `200`.
+// More information on metadata service can be found here - https://cloud.google.com/compute/docs/storing-retrieving-metadata
 func (g *containerRegistryProvider) Enabled() bool {
-	value, err := credentialprovider.ReadUrl(metadataScopes+"?alt=json", g.Client, metadataHeader)
-	if err != nil {
+	if !onGCEVM() {
 		return false
 	}
+	// Given that we are on GCE, we should keep retrying until the metadata server responds.
+	value := runWithBackoff(func() ([]byte, error) {
+		value, err := credentialprovider.ReadUrl(serviceAccounts, g.Client, metadataHeader)
+		if err != nil {
+			glog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err)
+		}
+		return value, err
+	})
+	// We expect the service account to return a list of account directories separated by newlines, e.g.,
+	//   sv-account-name1/
+	//   sv-account-name2/
+	// ref: https://cloud.google.com/compute/docs/storing-retrieving-metadata
+	defaultServiceAccountExists := false
+	for _, sa := range strings.Split(string(value), "\n") {
+		if strings.TrimSpace(sa) == defaultServiceAccount {
+			defaultServiceAccountExists = true
+			break
+		}
+	}
+	if !defaultServiceAccountExists {
+		glog.V(2).Infof("'default' service account does not exist. Found following service accounts: %q", string(value))
+		return false
+	}
+	url := metadataScopes + "?alt=json"
+	value = runWithBackoff(func() ([]byte, error) {
+		value, err := credentialprovider.ReadUrl(url, g.Client, metadataHeader)
+		if err != nil {
+			glog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err)
+		}
+		return value, err
+	})
 	var scopes []string
-	if err := json.Unmarshal([]byte(value), &scopes); err != nil {
+	if err := json.Unmarshal(value, &scopes); err != nil {
+		glog.Errorf("Failed to unmarshal scopes: %v", err)
 		return false
 	}
-
 	for _, v := range scopes {
 		// cloudPlatformScope implies storage scope.
 		if strings.HasPrefix(v, storageScopePrefix) || strings.HasPrefix(v, cloudPlatformScopePrefix) {

pkg/credentialprovider/gcp/metadata_test.go

@@ -20,9 +20,11 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -31,6 +33,14 @@ import (
 	utilnet "k8s.io/kubernetes/pkg/util/net"
 )
 
+func createProductNameFile() (string, error) {
+	file, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temporary test file: %v", err)
+	}
+	return file.Name(), ioutil.WriteFile(file.Name(), []byte("Google"), 0600)
+}
+
 func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
 	registryUrl := "hello.kubernetes.io"
 	email := "foo@bar.baz"
@@ -44,6 +54,12 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
    }
 }`, registryUrl, email, auth)
 
+	var err error
+	gceProductNameFile, err = createProductNameFile()
+	if err != nil {
+		t.Errorf("failed to create gce product name file: %v", err)
+	}
+	defer os.Remove(gceProductNameFile)
 	const probeEndpoint = "/computeMetadata/v1/"
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Only serve the one metadata key.
@@ -111,6 +127,12 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) {
    }
 }`, registryUrl, email, auth)
 
+	var err error
+	gceProductNameFile, err = createProductNameFile()
+	if err != nil {
+		t.Errorf("failed to create gce product name file: %v", err)
+	}
+	defer os.Remove(gceProductNameFile)
 	const probeEndpoint = "/computeMetadata/v1/"
 	const valueEndpoint = "/my/value"
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -176,11 +198,19 @@ func TestContainerRegistryBasics(t *testing.T) {
 	token := &tokenBlob{AccessToken: "ya26.lots-of-indiscernible-garbage"}
 
 	const (
-		defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
-		scopeEndpoint   = defaultEndpoint + "scopes"
-		emailEndpoint   = defaultEndpoint + "email"
-		tokenEndpoint   = defaultEndpoint + "token"
+		serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
+		defaultEndpoint         = "/computeMetadata/v1/instance/service-accounts/default/"
+		scopeEndpoint           = defaultEndpoint + "scopes"
+		emailEndpoint           = defaultEndpoint + "email"
+		tokenEndpoint           = defaultEndpoint + "token"
 	)
+	var err error
+	gceProductNameFile, err = createProductNameFile()
+	if err != nil {
+		t.Errorf("failed to create gce product name file: %v", err)
+	}
+	defer os.Remove(gceProductNameFile)
+
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Only serve the URL key and the value endpoint
 		if scopeEndpoint == r.URL.Path {
@@ -198,6 +228,9 @@ func TestContainerRegistryBasics(t *testing.T) {
 				t.Fatalf("unexpected error: %v", err)
 			}
 			fmt.Fprintln(w, string(bytes))
+		} else if serviceAccountsEndpoint == r.URL.Path {
+			w.WriteHeader(http.StatusOK)
+			fmt.Fprintln(w, "default/\ncustom")
 		} else {
 			w.WriteHeader(http.StatusNotFound)
 		}
@@ -243,10 +276,54 @@ func TestContainerRegistryBasics(t *testing.T) {
 	}
 }
 
+func TestContainerRegistryNoServiceAccount(t *testing.T) {
+	const (
+		serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
+	)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Only serve the URL key and the value endpoint
+		if serviceAccountsEndpoint == r.URL.Path {
+			w.WriteHeader(http.StatusOK)
+			w.Header().Set("Content-Type", "application/json")
+			bytes, err := json.Marshal([]string{})
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			fmt.Fprintln(w, string(bytes))
+		} else {
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer server.Close()
+
+	var err error
+	gceProductNameFile, err = createProductNameFile()
+	if err != nil {
+		t.Errorf("failed to create gce product name file: %v", err)
+	}
+	defer os.Remove(gceProductNameFile)
+
+	// Make a transport that reroutes all traffic to the example server
+	transport := utilnet.SetTransportDefaults(&http.Transport{
+		Proxy: func(req *http.Request) (*url.URL, error) {
+			return url.Parse(server.URL + req.URL.Path)
+		},
+	})
+
+	provider := &containerRegistryProvider{
+		metadataProvider{Client: &http.Client{Transport: transport}},
+	}
+
+	if provider.Enabled() {
+		t.Errorf("Provider is unexpectedly enabled")
+	}
+}
+
 func TestContainerRegistryNoStorageScope(t *testing.T) {
 	const (
-		defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
-		scopeEndpoint   = defaultEndpoint + "scopes"
+		serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
+		defaultEndpoint         = "/computeMetadata/v1/instance/service-accounts/default/"
+		scopeEndpoint           = defaultEndpoint + "scopes"
 	)
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Only serve the URL key and the value endpoint
@@ -254,12 +331,22 @@ func TestContainerRegistryNoStorageScope(t *testing.T) {
 			w.WriteHeader(http.StatusOK)
 			w.Header().Set("Content-Type", "application/json")
 			fmt.Fprint(w, `["https://www.googleapis.com/auth/compute.read_write"]`)
+		} else if serviceAccountsEndpoint == r.URL.Path {
+			w.WriteHeader(http.StatusOK)
+			fmt.Fprintln(w, "default/\ncustom")
 		} else {
 			w.WriteHeader(http.StatusNotFound)
 		}
 	}))
 	defer server.Close()
 
+	var err error
+	gceProductNameFile, err = createProductNameFile()
+	if err != nil {
+		t.Errorf("failed to create gce product name file: %v", err)
+	}
+	defer os.Remove(gceProductNameFile)
+
 	// Make a transport that reroutes all traffic to the example server
 	transport := utilnet.SetTransportDefaults(&http.Transport{
 		Proxy: func(req *http.Request) (*url.URL, error) {
@@ -278,8 +365,9 @@ func TestContainerRegistryNoStorageScope(t *testing.T) {
 
 func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
 	const (
-		defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
-		scopeEndpoint   = defaultEndpoint + "scopes"
+		serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
+		defaultEndpoint         = "/computeMetadata/v1/instance/service-accounts/default/"
+		scopeEndpoint           = defaultEndpoint + "scopes"
 	)
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Only serve the URL key and the value endpoint
@@ -287,12 +375,23 @@ func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
 			w.WriteHeader(http.StatusOK)
 			w.Header().Set("Content-Type", "application/json")
 			fmt.Fprint(w, `["https://www.googleapis.com/auth/compute.read_write","https://www.googleapis.com/auth/cloud-platform.read-only"]`)
+		} else if serviceAccountsEndpoint == r.URL.Path {
+			w.WriteHeader(http.StatusOK)
+			w.Header().Set("Content-Type", "application/json")
+			fmt.Fprintln(w, "default/\ncustom")
 		} else {
 			w.WriteHeader(http.StatusNotFound)
 		}
 	}))
 	defer server.Close()
 
+	var err error
+	gceProductNameFile, err = createProductNameFile()
+	if err != nil {
+		t.Errorf("failed to create gce product name file: %v", err)
+	}
+	defer os.Remove(gceProductNameFile)
+
 	// Make a transport that reroutes all traffic to the example server
 	transport := utilnet.SetTransportDefaults(&http.Transport{
 		Proxy: func(req *http.Request) (*url.URL, error) {

pkg/credentialprovider/provider.go

@@ -29,7 +29,11 @@ import (
 // DockerConfigProvider is the interface that registered extensions implement
 // to materialize 'dockercfg' credentials.
 type DockerConfigProvider interface {
+	// Enabled returns true if the config provider is enabled.
+	// Implementations can be blocking - e.g. metadata server unavailable.
 	Enabled() bool
+	// Provide returns docker configuration.
+	// Implementations can be blocking - e.g. metadata server unavailable.
 	Provide() DockerConfig
 	// LazyProvide() gets called after URL matches have been performed, so the
 	// location used as the key in DockerConfig would be redundant.