Allow cross-region image pulling with AWS' ECR

This is step two. We now create long-lived, lazy ECR providers in all regions.
When first used, they will create the actual ECR providers doing the work
behind the scenes, namely talking to ECR in the region where the image lives,
rather than the one our instance is running in.

Also:

- moved the list of AWS regions out of the AWS cloudprovider and into the
credentialprovider, then exported it from there.
- improved logging

Behold, running in us-east-1:

```
aws_credentials.go:127] Creating ecrProvider for us-west-2
aws_credentials.go:63] AWS request: ecr:GetAuthorizationToken in us-west-2
aws_credentials.go:217] Adding credentials for user AWS in us-west-2
Successfully pulled image 123456789012.dkr.ecr.us-west-2.amazonaws.com/test:latest"
```

*"One small step for a pod, one giant leap for Kube-kind."*

pkg/cloudprovider/providers/aws/aws.go

@@ -566,21 +566,7 @@ func getAvailabilityZone(metadata EC2Metadata) (string, error) {
 }
 
 func isRegionValid(region string) bool {
-	regions := [...]string{
-		"us-east-1",
-		"us-west-1",
-		"us-west-2",
-		"eu-west-1",
-		"eu-central-1",
-		"ap-southeast-1",
-		"ap-southeast-2",
-		"ap-northeast-1",
-		"ap-northeast-2",
-		"cn-north-1",
-		"us-gov-west-1",
-		"sa-east-1",
-	}
-	for _, r := range regions {
+	for _, r := range aws_credentials.AWSRegions {
 		if r == region {
 			return true
 		}

pkg/credentialprovider/aws/aws_credentials.go

@@ -18,6 +18,7 @@ package aws_credentials
 
 import (
 	"encoding/base64"
+	"fmt"
 	"strings"
 	"time"
 
@@ -26,23 +27,39 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/golang/glog"
-	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/credentialprovider"
 )
 
-var registryUrls = []string{"*.dkr.ecr.*.amazonaws.com"}
+// AWSRegions is the complete list of regions known to the AWS cloudprovider
+// and credentialprovider.
+var AWSRegions = [...]string{
+	"us-east-1",
+	"us-west-1",
+	"us-west-2",
+	"eu-west-1",
+	"eu-central-1",
+	"ap-southeast-1",
+	"ap-southeast-2",
+	"ap-northeast-1",
+	"cn-north-1",
+	"us-gov-west-1",
+	"sa-east-1",
+}
+
+const registryURLTemplate = "*.dkr.ecr.%s.amazonaws.com"
 
 // awsHandlerLogger is a handler that logs all AWS SDK requests
 // Copied from cloudprovider/aws/log_handler.go
 func awsHandlerLogger(req *request.Request) {
 	service := req.ClientInfo.ServiceName
+	region := req.Config.Region
 
 	name := "?"
 	if req.Operation != nil {
 		name = req.Operation.Name
 	}
 
-	glog.V(4).Infof("AWS request: %s %s", service, name)
+	glog.V(3).Infof("AWS request: %s:%s in %s", service, name, *region)
 }
 
 // An interface for testing purposes.
@@ -59,22 +76,81 @@ func (p *ecrTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenI
 	return p.svc.GetAuthorizationToken(input)
 }
 
+// lazyEcrProvider is a DockerConfigProvider that creates on demand an
+// ecrProvider for a given region and then proxies requests to it.
+type lazyEcrProvider struct {
+	region         string
+	regionURL      string
+	actualProvider *credentialprovider.CachingDockerConfigProvider
+}
+
 // ecrProvider is a DockerConfigProvider that gets and refreshes 12-hour tokens
 // from AWS to access ECR.
 type ecrProvider struct {
-	getter tokenGetter
+	region    string
+	regionURL string
+	getter    tokenGetter
 }
 
+// Init creates a lazy provider for each AWS region, in order to support
+// cross-region ECR access. They have to be lazy because it's unlikely, but not
+// impossible, that we'll use more than one.
 // Not using the package init() function: this module should be initialized only
 // if using the AWS cloud provider. This way, we avoid timeouts waiting for a
 // non-existent provider.
 func Init() {
-	credentialprovider.RegisterCredentialProvider("aws-ecr-key",
-		&credentialprovider.CachingDockerConfigProvider{
-			Provider: &ecrProvider{},
-			// Refresh credentials a little earlier before they expire
+	for _, region := range AWSRegions {
+		credentialprovider.RegisterCredentialProvider("aws-ecr-"+region,
+			&credentialprovider.CachingDockerConfigProvider{
+				Provider: &lazyEcrProvider{
+					region:    region,
+					regionURL: fmt.Sprintf(registryURLTemplate, region),
+				},
+				// This is going to be just a lazy proxy to the real ecrProvider.
+				// It holds no real credentials, so refresh practically never.
+				Lifetime: 365 * 24 * time.Hour,
+			})
+	}
+
+}
+
+// Enabled implements DockerConfigProvider.Enabled for the lazy provider.
+func (p *lazyEcrProvider) Enabled() bool {
+	return true
+}
+
+// LazyProvide implements DockerConfigProvider.LazyProvide. It will be called
+// by the client when attempting to pull an image and it will create the actual
+// provider only when we actually need it the first time.
+func (p *lazyEcrProvider) LazyProvide() *credentialprovider.DockerConfigEntry {
+	if p.actualProvider == nil {
+		glog.V(2).Infof("Creating ecrProvider for %s", p.region)
+		p.actualProvider = &credentialprovider.CachingDockerConfigProvider{
+			Provider: &ecrProvider{
+				region: p.region,
+				regionURL: p.regionURL,
+			},
+			// Refresh credentials a little earlier than expiration time
 			Lifetime: 11*time.Hour + 55*time.Minute,
-		})
+		}
+		if !p.actualProvider.Enabled() {
+
+			return nil
+		}
+	}
+	entry := p.actualProvider.Provide()[p.regionURL]
+	return &entry
+}
+
+// Provide implements DockerConfigProvider.Provide, creating dummy credentials.
+// Client code will call Provider.LazyProvide() at image pulling time.
+func (p *lazyEcrProvider) Provide() credentialprovider.DockerConfig {
+	entry := credentialprovider.DockerConfigEntry{
+		Provider: p,
+	}
+	cfg := credentialprovider.DockerConfig{}
+	cfg[p.regionURL] = entry
+	return cfg
 }
 
 // Enabled implements DockerConfigProvider.Enabled for the AWS token-based implementation.
@@ -82,33 +158,14 @@ func Init() {
 // TODO: figure how to enable it manually for deployments that are not on AWS but still
 // use ECR somehow?
 func (p *ecrProvider) Enabled() bool {
-	provider, err := cloudprovider.GetCloudProvider("aws", nil)
-	if err != nil {
-		glog.Errorf("while initializing AWS cloud provider %v", err)
-		return false
-	}
-	if provider == nil {
-		return false
-	}
-
-	zones, ok := provider.Zones()
-	if !ok {
-		glog.Errorf("couldn't get Zones() interface")
-		return false
-	}
-	zone, err := zones.GetZone()
-	if err != nil {
-		glog.Errorf("while getting zone %v", err)
-		return false
-	}
-	if zone.Region == "" {
-		glog.Errorf("Region information is empty")
+	if p.region == "" {
+		glog.Errorf("Called ecrProvider.Enabled() with no region set")
 		return false
 	}
 
 	getter := &ecrTokenGetter{svc: ecr.New(session.New(&aws.Config{
 		Credentials: nil,
-		Region:      &zone.Region,
+		Region:      &p.region,
 	}))}
 	getter.svc.Handlers.Sign.PushFrontNamed(request.NamedHandler{
 		Name: "k8s/logger",
@@ -158,10 +215,10 @@ func (p *ecrProvider) Provide() credentialprovider.DockerConfig {
 				Email: "not@val.id",
 			}
 
-			// Add our entry for each of the supported container registry URLs
-			for _, k := range registryUrls {
-				cfg[k] = entry
-			}
+			glog.V(3).Infof("Adding credentials for user %s in %s", user, p.region)
+			// Add our config entry for this region's registry URLs
+			cfg[p.regionURL] = entry
+
 		}
 	}
 	return cfg

pkg/credentialprovider/aws/aws_credentials_test.go

@@ -58,12 +58,15 @@ func (p *testTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationToken
 
 func TestEcrProvide(t *testing.T) {
 	registry := "123456789012.dkr.ecr.lala-land-1.amazonaws.com"
-	otherRegistries := []string{"private.registry.com",
+	otherRegistries := []string{
+		"private.registry.com",
 		"gcr.io",
 	}
 	image := "foo/bar"
 
 	provider := &ecrProvider{
+		region: "lala-land-1",
+		regionURL: "*.dkr.ecr.lala-land-1.amazonaws.com",
 		getter: &testTokenGetter{
 			user:     user,
 			password: password,