From 7eafa215f55a4b4f143f81bee5fc493250ffdd61 Mon Sep 17 00:00:00 2001 From: Dong Liu Date: Tue, 19 Dec 2017 15:18:37 +0800 Subject: [PATCH] Split auth related config for Azure --- pkg/cloudprovider/providers/azure/BUILD | 9 +- pkg/cloudprovider/providers/azure/auth/BUILD | 28 ++++ .../providers/azure/auth/azure_auth.go | 124 +++++++++++++++++ pkg/cloudprovider/providers/azure/azure.go | 129 +++--------------- .../providers/azure/azure_controllerCommon.go | 1 - .../providers/azure/azure_test.go | 7 +- pkg/credentialprovider/azure/BUILD | 3 +- .../azure/azure_credentials.go | 40 +++++- 8 files changed, 218 insertions(+), 123 deletions(-) create mode 100644 pkg/cloudprovider/providers/azure/auth/BUILD create mode 100644 pkg/cloudprovider/providers/azure/auth/azure_auth.go diff --git a/pkg/cloudprovider/providers/azure/BUILD b/pkg/cloudprovider/providers/azure/BUILD index accc2594e30..fe131367e0c 100644 --- a/pkg/cloudprovider/providers/azure/BUILD +++ b/pkg/cloudprovider/providers/azure/BUILD @@ -32,6 +32,7 @@ go_library( deps = [ "//pkg/api/v1/service:go_default_library", "//pkg/cloudprovider:go_default_library", + "//pkg/cloudprovider/providers/azure/auth:go_default_library", "//pkg/controller:go_default_library", "//pkg/version:go_default_library", "//pkg/volume:go_default_library", @@ -41,13 +42,11 @@ go_library( "//vendor/github.com/Azure/azure-sdk-for-go/arm/storage:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/storage:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", - "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/rubiojr/go-vhd/vhd:go_default_library", - "//vendor/golang.org/x/crypto/pkcs12:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", @@ -71,6 +70,7 @@ go_test( importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/azure", deps = [ "//pkg/api/v1/service:go_default_library", + "//pkg/cloudprovider/providers/azure/auth:go_default_library", "//pkg/kubelet/apis:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/arm/compute:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/arm/network:go_default_library", @@ -93,6 +93,9 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//pkg/cloudprovider/providers/azure/auth:all-srcs", + ], tags = ["automanaged"], ) diff --git a/pkg/cloudprovider/providers/azure/auth/BUILD b/pkg/cloudprovider/providers/azure/auth/BUILD new file mode 100644 index 00000000000..cc733d385aa --- /dev/null +++ b/pkg/cloudprovider/providers/azure/auth/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["azure_auth.go"], + importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/azure/auth", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library", + "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/golang.org/x/crypto/pkcs12:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/cloudprovider/providers/azure/auth/azure_auth.go b/pkg/cloudprovider/providers/azure/auth/azure_auth.go new file mode 100644 index 00000000000..948206f8eb6 --- /dev/null +++ b/pkg/cloudprovider/providers/azure/auth/azure_auth.go @@ -0,0 +1,124 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "crypto/rsa" + "crypto/x509" + "fmt" + "io/ioutil" + + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/golang/glog" + "golang.org/x/crypto/pkcs12" +) + +// AzureAuthConfig holds auth related part of cloud config +type AzureAuthConfig struct { + // The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13 + Cloud string `json:"cloud" yaml:"cloud"` + // The AAD Tenant ID for the Subscription that the cluster is deployed in + TenantID string `json:"tenantId" yaml:"tenantId"` + // The ClientID for an AAD application with RBAC access to talk to Azure RM APIs + AADClientID string `json:"aadClientId" yaml:"aadClientId"` + // The ClientSecret for an AAD application with RBAC access to talk to Azure RM APIs + AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` + // The path of a client certificate for an AAD application with RBAC access to talk to Azure RM APIs + AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"` + // The password of the client certificate for an AAD application with RBAC access to talk to Azure RM APIs + AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"` + // Use managed service identity for the virtual machine to access Azure ARM APIs + UseManagedIdentityExtension bool `json:"useManagedIdentityExtension"` + // The ID of the Azure Subscription that the cluster is deployed in + SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"` +} + +// GetServicePrincipalToken creates a new service principal token based on the configuration +func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) { + oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID) + if err != nil { + return nil, fmt.Errorf("creating the OAuth config: %v", err) + } + + if config.UseManagedIdentityExtension { + glog.V(2).Infoln("azure: using managed identity extension to retrieve access token") + msiEndpoint, err := adal.GetMSIVMEndpoint() + if err != nil { + return nil, fmt.Errorf("Getting the managed service identity endpoint: %v", err) + } + return adal.NewServicePrincipalTokenFromMSI( + msiEndpoint, + env.ServiceManagementEndpoint) + } + + if len(config.AADClientSecret) > 0 { + glog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token") + return adal.NewServicePrincipalToken( + *oauthConfig, + config.AADClientID, + config.AADClientSecret, + env.ServiceManagementEndpoint) + } + + if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 { + glog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token") + certData, err := ioutil.ReadFile(config.AADClientCertPath) + if err != nil { + return nil, fmt.Errorf("reading the client certificate from file %s: %v", config.AADClientCertPath, err) + } + certificate, privateKey, err := decodePkcs12(certData, config.AADClientCertPassword) + if err != nil { + return nil, fmt.Errorf("decoding the client certificate: %v", err) + } + return adal.NewServicePrincipalTokenFromCertificate( + *oauthConfig, + config.AADClientID, + certificate, + privateKey, + env.ServiceManagementEndpoint) + } + + return nil, fmt.Errorf("No credentials provided for AAD application %s", config.AADClientID) +} + +// ParseAzureEnvironment returns azure environment by name +func ParseAzureEnvironment(cloudName string) (*azure.Environment, error) { + var env azure.Environment + var err error + if cloudName == "" { + env = azure.PublicCloud + } else { + env, err = azure.EnvironmentFromName(cloudName) + } + return &env, err +} + +// decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and +// the private RSA key +func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) { + privateKey, certificate, err := pkcs12.Decode(pkcs, password) + if err != nil { + return nil, nil, fmt.Errorf("decoding the PKCS#12 client certificate: %v", err) + } + rsaPrivateKey, isRsaKey := privateKey.(*rsa.PrivateKey) + if !isRsaKey { + return nil, nil, fmt.Errorf("PKCS#12 certificate must contain a RSA private key") + } + + return certificate, rsaPrivateKey, nil +} diff --git a/pkg/cloudprovider/providers/azure/azure.go b/pkg/cloudprovider/providers/azure/azure.go index 936c3cfbb67..170091cace9 100644 --- a/pkg/cloudprovider/providers/azure/azure.go +++ b/pkg/cloudprovider/providers/azure/azure.go @@ -17,16 +17,16 @@ limitations under the License. package azure import ( - "crypto/rsa" - "crypto/x509" "fmt" "io" "io/ioutil" "strings" "time" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/flowcontrol" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/azure/auth" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/version" @@ -35,12 +35,9 @@ import ( "github.com/Azure/azure-sdk-for-go/arm/network" "github.com/Azure/azure-sdk-for-go/arm/storage" "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" "github.com/ghodss/yaml" "github.com/golang/glog" - "golang.org/x/crypto/pkcs12" - "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -61,12 +58,8 @@ const ( // Config holds the configuration parsed from the --cloud-config flag // All fields are required unless otherwise specified type Config struct { - // The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13 - Cloud string `json:"cloud" yaml:"cloud"` - // The AAD Tenant ID for the Subscription that the cluster is deployed in - TenantID string `json:"tenantId" yaml:"tenantId"` - // The ID of the Azure Subscription that the cluster is deployed in - SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"` + auth.AzureAuthConfig + // The name of the resource group that the cluster is deployed in ResourceGroup string `json:"resourceGroup" yaml:"resourceGroup"` // The location of the resource group that the cluster is deployed in @@ -96,15 +89,6 @@ type Config struct { // the cloudprovider will try to add all nodes to a single backend pool which is forbidden. // In other words, if you use multiple agent pools (scale sets), you MUST set this field. PrimaryScaleSetName string `json:"primaryScaleSetName" yaml:"primaryScaleSetName"` - - // The ClientID for an AAD application with RBAC access to talk to Azure RM APIs - AADClientID string `json:"aadClientId" yaml:"aadClientId"` - // The ClientSecret for an AAD application with RBAC access to talk to Azure RM APIs - AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` - // The path of a client certificate for an AAD application with RBAC access to talk to Azure RM APIs - AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"` - // The password of the client certificate for an AAD application with RBAC access to talk to Azure RM APIs - AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"` // Enable exponential backoff to manage resource request retries CloudProviderBackoff bool `json:"cloudProviderBackoff" yaml:"cloudProviderBackoff"` // Backoff retry limit @@ -122,9 +106,6 @@ type Config struct { // Rate limit Bucket Size CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket" yaml:"cloudProviderRateLimitBucket"` - // Use managed service identity for the virtual machine to access Azure ARM APIs - UseManagedIdentityExtension bool `json:"useManagedIdentityExtension"` - // Maximum allowed LoadBalancer Rule Count is the limit enforced by Azure Load balancer MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount"` } @@ -226,81 +207,24 @@ func init() { cloudprovider.RegisterCloudProvider(CloudProviderName, NewCloud) } -// decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and -// the private RSA key -func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) { - privateKey, certificate, err := pkcs12.Decode(pkcs, password) - if err != nil { - return nil, nil, fmt.Errorf("decoding the PKCS#12 client certificate: %v", err) - } - rsaPrivateKey, isRsaKey := privateKey.(*rsa.PrivateKey) - if !isRsaKey { - return nil, nil, fmt.Errorf("PKCS#12 certificate must contain a RSA private key") - } - - return certificate, rsaPrivateKey, nil -} - -// GetServicePrincipalToken creates a new service principal token based on the configuration -func GetServicePrincipalToken(config *Config, env *azure.Environment) (*adal.ServicePrincipalToken, error) { - oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID) - if err != nil { - return nil, fmt.Errorf("creating the OAuth config: %v", err) - } - - if config.UseManagedIdentityExtension { - glog.V(2).Infoln("azure: using managed identity extension to retrieve access token") - msiEndpoint, err := adal.GetMSIVMEndpoint() - if err != nil { - return nil, fmt.Errorf("Getting the managed service identity endpoint: %v", err) - } - return adal.NewServicePrincipalTokenFromMSI( - msiEndpoint, - env.ServiceManagementEndpoint) - } - - if len(config.AADClientSecret) > 0 { - glog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token") - return adal.NewServicePrincipalToken( - *oauthConfig, - config.AADClientID, - config.AADClientSecret, - env.ServiceManagementEndpoint) - } - - if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 { - glog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token") - certData, err := ioutil.ReadFile(config.AADClientCertPath) - if err != nil { - return nil, fmt.Errorf("reading the client certificate from file %s: %v", config.AADClientCertPath, err) - } - certificate, privateKey, err := decodePkcs12(certData, config.AADClientCertPassword) - if err != nil { - return nil, fmt.Errorf("decoding the client certificate: %v", err) - } - return adal.NewServicePrincipalTokenFromCertificate( - *oauthConfig, - config.AADClientID, - certificate, - privateKey, - env.ServiceManagementEndpoint) - } - - return nil, fmt.Errorf("No credentials provided for AAD application %s", config.AADClientID) -} - // NewCloud returns a Cloud with initialized clients func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) { - config, env, err := ParseConfig(configReader) + config, err := parseConfig(configReader) if err != nil { return nil, err } + + env, err := auth.ParseAzureEnvironment(config.Cloud) + if err != nil { + return nil, err + } + az := Cloud{ Config: *config, Environment: *env, } - servicePrincipalToken, err := GetServicePrincipalToken(config, env) + servicePrincipalToken, err := auth.GetServicePrincipalToken(&config.AzureAuthConfig, env) if err != nil { return nil, err } @@ -433,7 +357,7 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) { az.MaximumLoadBalancerRuleCount = maximumLoadBalancerRuleCount } - if az.Config.VMType == vmTypeVMSS { + if strings.EqualFold(vmTypeVMSS, az.Config.VMType) { az.vmSet = newScaleSet(&az) } else { az.vmSet = newAvailabilitySet(&az) @@ -445,38 +369,24 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) { return &az, nil } -// ParseConfig returns a parsed configuration and azure.Environment for an Azure cloudprovider config file -func ParseConfig(configReader io.Reader) (*Config, *azure.Environment, error) { +// parseConfig returns a parsed configuration for an Azure cloudprovider config file +func parseConfig(configReader io.Reader) (*Config, error) { var config Config - var env azure.Environment if configReader == nil { - return &config, &env, nil + return &config, nil } configContents, err := ioutil.ReadAll(configReader) if err != nil { - return nil, nil, err + return nil, err } err = yaml.Unmarshal(configContents, &config) if err != nil { - return nil, nil, err + return nil, err } - if config.Cloud == "" { - env = azure.PublicCloud - } else { - env, err = azure.EnvironmentFromName(config.Cloud) - if err != nil { - return nil, nil, err - } - } - - if config.VMType != "" { - config.VMType = strings.ToLower(config.VMType) - } - - return &config, &env, nil + return &config, nil } // Initialize passes a Kubernetes clientBuilder interface to the cloud provider @@ -538,7 +448,6 @@ func initDiskControllers(az *Cloud) error { storageEndpointSuffix: az.Environment.StorageEndpointSuffix, managementEndpoint: az.Environment.ResourceManagerEndpoint, resourceGroup: az.ResourceGroup, - tenantID: az.TenantID, tokenEndPoint: az.Environment.ActiveDirectoryEndpoint, subscriptionID: az.SubscriptionID, cloud: az, diff --git a/pkg/cloudprovider/providers/azure/azure_controllerCommon.go b/pkg/cloudprovider/providers/azure/azure_controllerCommon.go index fdb78e2af7b..c445c2783ec 100644 --- a/pkg/cloudprovider/providers/azure/azure_controllerCommon.go +++ b/pkg/cloudprovider/providers/azure/azure_controllerCommon.go @@ -53,7 +53,6 @@ var defaultBackOff = kwait.Backoff{ } type controllerCommon struct { - tenantID string subscriptionID string location string storageEndpointSuffix string diff --git a/pkg/cloudprovider/providers/azure/azure_test.go b/pkg/cloudprovider/providers/azure/azure_test.go index 91b1d48549f..47181bb0d3f 100644 --- a/pkg/cloudprovider/providers/azure/azure_test.go +++ b/pkg/cloudprovider/providers/azure/azure_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/flowcontrol" serviceapi "k8s.io/kubernetes/pkg/api/v1/service" + "k8s.io/kubernetes/pkg/cloudprovider/providers/azure/auth" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "github.com/Azure/azure-sdk-for-go/arm/compute" @@ -846,8 +847,10 @@ func TestReconcilePublicIPWithExternalAndInternalSwitch(t *testing.T) { func getTestCloud() (az *Cloud) { az = &Cloud{ Config: Config{ - TenantID: "tenant", - SubscriptionID: "subscription", + AzureAuthConfig: auth.AzureAuthConfig{ + TenantID: "tenant", + SubscriptionID: "subscription", + }, ResourceGroup: "rg", VnetResourceGroup: "rg", Location: "westus", diff --git a/pkg/credentialprovider/azure/BUILD b/pkg/credentialprovider/azure/BUILD index c7213861b7f..afb23f31b7d 100644 --- a/pkg/credentialprovider/azure/BUILD +++ b/pkg/credentialprovider/azure/BUILD @@ -14,13 +14,14 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/credentialprovider/azure", deps = [ - "//pkg/cloudprovider/providers/azure:go_default_library", + "//pkg/cloudprovider/providers/azure/auth:go_default_library", "//pkg/credentialprovider:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/arm/containerregistry:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library", "//vendor/github.com/dgrijalva/jwt-go:go_default_library", + "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", ], diff --git a/pkg/credentialprovider/azure/azure_credentials.go b/pkg/credentialprovider/azure/azure_credentials.go index ff251497ce3..e48d8133f55 100644 --- a/pkg/credentialprovider/azure/azure_credentials.go +++ b/pkg/credentialprovider/azure/azure_credentials.go @@ -18,16 +18,18 @@ package azure import ( "io" + "io/ioutil" "os" "time" "github.com/Azure/azure-sdk-for-go/arm/containerregistry" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/adal" - azureapi "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/ghodss/yaml" "github.com/golang/glog" "github.com/spf13/pflag" - "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" + "k8s.io/kubernetes/pkg/cloudprovider/providers/azure/auth" "k8s.io/kubernetes/pkg/credentialprovider" ) @@ -60,18 +62,44 @@ func NewACRProvider(configFile *string) credentialprovider.DockerConfigProvider type acrProvider struct { file *string - config *azure.Config - environment *azureapi.Environment + config *auth.AzureAuthConfig + environment *azure.Environment registryClient RegistriesClient servicePrincipalToken *adal.ServicePrincipalToken } +// ParseConfig returns a parsed configuration for an Azure cloudprovider config file +func parseConfig(configReader io.Reader) (*auth.AzureAuthConfig, error) { + var config auth.AzureAuthConfig + + if configReader == nil { + return &config, nil + } + + configContents, err := ioutil.ReadAll(configReader) + if err != nil { + return nil, err + } + err = yaml.Unmarshal(configContents, &config) + if err != nil { + return nil, err + } + + return &config, nil +} + func (a *acrProvider) loadConfig(rdr io.Reader) error { var err error - a.config, a.environment, err = azure.ParseConfig(rdr) + a.config, err = parseConfig(rdr) if err != nil { glog.Errorf("Failed to load azure credential file: %v", err) } + + a.environment, err = auth.ParseAzureEnvironment(a.config.Cloud) + if err != nil { + return err + } + return nil } @@ -94,7 +122,7 @@ func (a *acrProvider) Enabled() bool { return false } - a.servicePrincipalToken, err = azure.GetServicePrincipalToken(a.config, a.environment) + a.servicePrincipalToken, err = auth.GetServicePrincipalToken(a.config, a.environment) if err != nil { glog.Errorf("Failed to create service principal token: %v", err) return false