diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index 1ecf9a7e74c..e78744b03b3 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -42,7 +42,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/cloudprovider" - "k8s.io/kubernetes/pkg/credentialprovider/aws" + aws_credentials "k8s.io/kubernetes/pkg/credentialprovider/aws" "k8s.io/kubernetes/pkg/types" "github.com/golang/glog" @@ -585,21 +585,7 @@ func getAvailabilityZone(metadata EC2Metadata) (string, error) { } func isRegionValid(region string) bool { - regions := [...]string{ - "us-east-1", - "us-west-1", - "us-west-2", - "eu-west-1", - "eu-central-1", - "ap-southeast-1", - "ap-southeast-2", - "ap-northeast-1", - "ap-northeast-2", - "cn-north-1", - "us-gov-west-1", - "sa-east-1", - } - for _, r := range regions { + for _, r := range aws_credentials.AWSRegions { if r == region { return true } diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go index aeb7316b328..3b9b5c9820d 100644 --- a/pkg/credentialprovider/aws/aws_credentials.go +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -14,10 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package aws_credentials +package credentials import ( "encoding/base64" + "fmt" "strings" "time" @@ -26,23 +27,40 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ecr" "github.com/golang/glog" - "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/credentialprovider" ) -var registryUrls = []string{"*.dkr.ecr.*.amazonaws.com"} +// AWSRegions is the complete list of regions known to the AWS cloudprovider +// and credentialprovider. +var AWSRegions = [...]string{ + "us-east-1", + "us-west-1", + "us-west-2", + "eu-west-1", + "eu-central-1", + "ap-southeast-1", + "ap-southeast-2", + "ap-northeast-1", + "ap-northeast-2", + "cn-north-1", + "us-gov-west-1", + "sa-east-1", +} + +const registryURLTemplate = "*.dkr.ecr.%s.amazonaws.com" // awsHandlerLogger is a handler that logs all AWS SDK requests // Copied from cloudprovider/aws/log_handler.go func awsHandlerLogger(req *request.Request) { service := req.ClientInfo.ServiceName + region := req.Config.Region name := "?" if req.Operation != nil { name = req.Operation.Name } - glog.V(4).Infof("AWS request: %s %s", service, name) + glog.V(3).Infof("AWS request: %s:%s in %s", service, name, *region) } // An interface for testing purposes. @@ -59,22 +77,86 @@ func (p *ecrTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenI return p.svc.GetAuthorizationToken(input) } +// lazyEcrProvider is a DockerConfigProvider that creates on demand an +// ecrProvider for a given region and then proxies requests to it. +type lazyEcrProvider struct { + region string + regionURL string + actualProvider *credentialprovider.CachingDockerConfigProvider +} + +var _ credentialprovider.DockerConfigProvider = &lazyEcrProvider{} + // ecrProvider is a DockerConfigProvider that gets and refreshes 12-hour tokens // from AWS to access ECR. type ecrProvider struct { - getter tokenGetter + region string + regionURL string + getter tokenGetter } +var _ credentialprovider.DockerConfigProvider = &ecrProvider{} + +// Init creates a lazy provider for each AWS region, in order to support +// cross-region ECR access. They have to be lazy because it's unlikely, but not +// impossible, that we'll use more than one. // Not using the package init() function: this module should be initialized only // if using the AWS cloud provider. This way, we avoid timeouts waiting for a // non-existent provider. func Init() { - credentialprovider.RegisterCredentialProvider("aws-ecr-key", - &credentialprovider.CachingDockerConfigProvider{ - Provider: &ecrProvider{}, - // Refresh credentials a little earlier before they expire + for _, region := range AWSRegions { + credentialprovider.RegisterCredentialProvider("aws-ecr-"+region, + &lazyEcrProvider{ + region: region, + regionURL: fmt.Sprintf(registryURLTemplate, region), + }) + } + +} + +// Enabled implements DockerConfigProvider.Enabled for the lazy provider. +// Since we perform no checks/work of our own and actualProvider is only created +// later at image pulling time (if ever), always return true. +func (p *lazyEcrProvider) Enabled() bool { + return true +} + +// LazyProvide implements DockerConfigProvider.LazyProvide. It will be called +// by the client when attempting to pull an image and it will create the actual +// provider only when we actually need it the first time. +func (p *lazyEcrProvider) LazyProvide() *credentialprovider.DockerConfigEntry { + if p.actualProvider == nil { + glog.V(2).Infof("Creating ecrProvider for %s", p.region) + p.actualProvider = &credentialprovider.CachingDockerConfigProvider{ + Provider: newEcrProvider(p.region, nil), + // Refresh credentials a little earlier than expiration time Lifetime: 11*time.Hour + 55*time.Minute, - }) + } + if !p.actualProvider.Enabled() { + return nil + } + } + entry := p.actualProvider.Provide()[p.regionURL] + return &entry +} + +// Provide implements DockerConfigProvider.Provide, creating dummy credentials. +// Client code will call Provider.LazyProvide() at image pulling time. +func (p *lazyEcrProvider) Provide() credentialprovider.DockerConfig { + entry := credentialprovider.DockerConfigEntry{ + Provider: p, + } + cfg := credentialprovider.DockerConfig{} + cfg[p.regionURL] = entry + return cfg +} + +func newEcrProvider(region string, getter tokenGetter) *ecrProvider { + return &ecrProvider{ + region: region, + regionURL: fmt.Sprintf(registryURLTemplate, region), + getter: getter, + } } // Enabled implements DockerConfigProvider.Enabled for the AWS token-based implementation. @@ -82,33 +164,14 @@ func Init() { // TODO: figure how to enable it manually for deployments that are not on AWS but still // use ECR somehow? func (p *ecrProvider) Enabled() bool { - provider, err := cloudprovider.GetCloudProvider("aws", nil) - if err != nil { - glog.Errorf("while initializing AWS cloud provider %v", err) - return false - } - if provider == nil { - return false - } - - zones, ok := provider.Zones() - if !ok { - glog.Errorf("couldn't get Zones() interface") - return false - } - zone, err := zones.GetZone() - if err != nil { - glog.Errorf("while getting zone %v", err) - return false - } - if zone.Region == "" { - glog.Errorf("Region information is empty") + if p.region == "" { + glog.Errorf("Called ecrProvider.Enabled() with no region set") return false } getter := &ecrTokenGetter{svc: ecr.New(session.New(&aws.Config{ Credentials: nil, - Region: &zone.Region, + Region: &p.region, }))} getter.svc.Handlers.Sign.PushFrontNamed(request.NamedHandler{ Name: "k8s/logger", @@ -158,10 +221,10 @@ func (p *ecrProvider) Provide() credentialprovider.DockerConfig { Email: "not@val.id", } - // Add our entry for each of the supported container registry URLs - for _, k := range registryUrls { - cfg[k] = entry - } + glog.V(3).Infof("Adding credentials for user %s in %s", user, p.region) + // Add our config entry for this region's registry URLs + cfg[p.regionURL] = entry + } } return cfg diff --git a/pkg/credentialprovider/aws/aws_credentials_test.go b/pkg/credentialprovider/aws/aws_credentials_test.go index a07493993f1..b286c7d61d2 100644 --- a/pkg/credentialprovider/aws/aws_credentials_test.go +++ b/pkg/credentialprovider/aws/aws_credentials_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package aws_credentials +package credentials import ( "encoding/base64" @@ -58,17 +58,18 @@ func (p *testTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationToken func TestEcrProvide(t *testing.T) { registry := "123456789012.dkr.ecr.lala-land-1.amazonaws.com" - otherRegistries := []string{"private.registry.com", + otherRegistries := []string{ + "private.registry.com", "gcr.io", } image := "foo/bar" - provider := &ecrProvider{ - getter: &testTokenGetter{ + provider := newEcrProvider("lala-land-1", + &testTokenGetter{ user: user, password: password, - endpoint: registry}, - } + endpoint: registry, + }) keyring := &credentialprovider.BasicDockerKeyring{} keyring.Add(provider.Provide())