Support cluster using network resources (VNet, LB, IP, etc.) across AAD Tenants.

staging/src/k8s.io/legacy-cloud-providers/azure/BUILD

@@ -83,6 +83,7 @@ go_library(
         "//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/storage:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/rubiojr/go-vhd/vhd:go_default_library",

staging/src/k8s.io/legacy-cloud-providers/azure/auth/azure_auth.go

@@ -68,9 +68,20 @@ type AzureAuthConfig struct {
 	// ResourceManagerEndpoint is the cloud's resource manager endpoint. If set, cloud provider queries this endpoint
 	// in order to generate an autorest.Environment instance instead of using one of the pre-defined Environments.
 	ResourceManagerEndpoint string `json:"resourceManagerEndpoint,omitempty" yaml:"resourceManagerEndpoint,omitempty"`
+	// The AAD Tenant ID for the Subscription that the network resources are deployed in
+	NetworkResourceTenantID string `json:"networkResourceTenantID,omitempty" yaml:"networkResourceTenantID,omitempty"`
+	// The ID of the Azure Subscription that the network resources are deployed in
+	NetworkResourceSubscriptionID string `json:"networkResourceSubscriptionID,omitempty" yaml:"networkResourceSubscriptionID,omitempty"`
 }
 
-// GetServicePrincipalToken creates a new service principal token based on the configuration
+// GetServicePrincipalToken creates a new service principal token based on the configuration.
+//
+// By default, the cluster and its network resources are deployed in the same AAD Tenant and Subscription,
+// and all azure clients use this method to fetch Service Principal Token.
+//
+// If NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
+// than only azure clients except VM/VMSS and network resource ones use this method to fetch Token.
+// For tokens for VM/VMSS and network resource ones, please check GetMultiTenantServicePrincipalToken and GetNetworkResourceServicePrincipalToken.
 func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
 	var tenantID string
 	if strings.EqualFold(config.IdentitySystem, ADFSIdentitySystem) {
@@ -132,6 +143,75 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (
 	return nil, ErrorNoAuth
 }
 
+// GetMultiTenantServicePrincipalToken is used when (and only when) NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID.
+//
+// In that scenario, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
+// and this method creates a new multi-tenant service principal token based on the configuration.
+//
+// PrimaryToken of the returned multi-tenant token is for the AAD Tenant specified by TenantID, and AuxiliaryToken of the returned multi-tenant token is for the AAD Tenant specified by NetworkResourceTenantID.
+//
+// Azure VM/VMSS clients use this multi-tenant token, in order to operate those VM/VMSS in AAD Tenant specified by TenantID, and meanwhile in their payload they are referencing network resources (e.g. Load Balancer, Network Security Group, etc.) in AAD Tenant specified by NetworkResourceTenantID.
+func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.MultiTenantServicePrincipalToken, error) {
+	err := config.checkConfigWhenNetworkResourceInDifferentTenant()
+	if err != nil {
+		return nil, fmt.Errorf("got error(%v) in getting multi-tenant service principal token", err)
+	}
+
+	multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(
+		env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("creating the multi-tenant OAuth config: %v", err)
+	}
+
+	if len(config.AADClientSecret) > 0 {
+		klog.V(2).Infoln("azure: using client_id+client_secret to retrieve multi-tenant access token")
+		return adal.NewMultiTenantServicePrincipalToken(
+			multiTenantOAuthConfig,
+			config.AADClientID,
+			config.AADClientSecret,
+			env.ServiceManagementEndpoint)
+	}
+
+	if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
+		return nil, fmt.Errorf("AAD Application client certificate authentication is not supported in getting multi-tenant service principal token")
+	}
+
+	return nil, ErrorNoAuth
+}
+
+// GetNetworkResourceServicePrincipalToken is used when (and only when) NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID.
+//
+// In that scenario, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
+// and this method creates a new service principal token for network resources tenant based on the configuration.
+//
+// Azure network resource (Load Balancer, Public IP, Route Table, Network Security Group and their sub level resources) clients use this multi-tenant token, in order to operate resources in AAD Tenant specified by NetworkResourceTenantID.
+func GetNetworkResourceServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
+	err := config.checkConfigWhenNetworkResourceInDifferentTenant()
+	if err != nil {
+		return nil, fmt.Errorf("got error(%v) in getting network resources service principal token", err)
+	}
+
+	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil)
+	if err != nil {
+		return nil, fmt.Errorf("creating the OAuth config for network resources tenant: %v", err)
+	}
+
+	if len(config.AADClientSecret) > 0 {
+		klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token for network resources tenant")
+		return adal.NewServicePrincipalToken(
+			*oauthConfig,
+			config.AADClientID,
+			config.AADClientSecret,
+			env.ServiceManagementEndpoint)
+	}
+
+	if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
+		return nil, fmt.Errorf("AAD Application client certificate authentication is not supported in getting network resources service principal token")
+	}
+
+	return nil, ErrorNoAuth
+}
+
 // ParseAzureEnvironment returns the azure environment.
 // If 'resourceManagerEndpoint' is set, the environment is computed by quering the cloud's resource manager endpoint.
 // Otherwise, a pre-defined Environment is looked up by name.
@@ -155,6 +235,16 @@ func ParseAzureEnvironment(cloudName, resourceManagerEndpoint, identitySystem st
 	return &env, err
 }
 
+// UsesNetworkResourceInDifferentTenant determines whether the AzureAuthConfig indicates to use network resources in different AAD Tenant and Subscription than those for the cluster
+// Return true only when both NetworkResourceTenantID and NetworkResourceSubscriptionID are specified
+// and they are not equals to TenantID and SubscriptionID
+func (config *AzureAuthConfig) UsesNetworkResourceInDifferentTenant() bool {
+	return len(config.NetworkResourceTenantID) > 0 &&
+		len(config.NetworkResourceSubscriptionID) > 0 &&
+		!strings.EqualFold(config.NetworkResourceTenantID, config.TenantID) &&
+		!strings.EqualFold(config.NetworkResourceSubscriptionID, config.SubscriptionID)
+}
+
 // decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and
 // the private RSA key
 func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) {
@@ -181,3 +271,20 @@ func azureStackOverrides(env *azure.Environment, resourceManagerEndpoint, identi
 		env.ActiveDirectoryEndpoint = strings.TrimSuffix(env.ActiveDirectoryEndpoint, "adfs")
 	}
 }
+
+// checkConfigWhenNetworkResourceInDifferentTenant checks configuration for the scenario of using network resource in different tenant
+func (config *AzureAuthConfig) checkConfigWhenNetworkResourceInDifferentTenant() error {
+	if !config.UsesNetworkResourceInDifferentTenant() {
+		return fmt.Errorf("NetworkResourceTenantID and NetworkResourceSubscriptionID must be configured")
+	}
+
+	if strings.EqualFold(config.IdentitySystem, ADFSIdentitySystem) {
+		return fmt.Errorf("ADFS identity system is not supported")
+	}
+
+	if config.UseManagedIdentityExtension {
+		return fmt.Errorf("managed identity is not supported")
+	}
+
+	return nil
+}

staging/src/k8s.io/legacy-cloud-providers/azure/auth/azure_auth_test.go

@@ -24,6 +24,32 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var (
+	CrossTenantNetworkResourceNegativeConfig = []*AzureAuthConfig{
+		{
+			TenantID:        "TenantID",
+			AADClientID:     "AADClientID",
+			AADClientSecret: "AADClientSecret",
+		},
+		{
+			TenantID:                      "TenantID",
+			AADClientID:                   "AADClientID",
+			AADClientSecret:               "AADClientSecret",
+			NetworkResourceTenantID:       "NetworkResourceTenantID",
+			NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID",
+			IdentitySystem:                ADFSIdentitySystem,
+		},
+		{
+			TenantID:                      "TenantID",
+			AADClientID:                   "AADClientID",
+			AADClientSecret:               "AADClientSecret",
+			NetworkResourceTenantID:       "NetworkResourceTenantID",
+			NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID",
+			UseManagedIdentityExtension:   true,
+		},
+	}
+)
+
 func TestGetServicePrincipalTokenFromMSIWithUserAssignedID(t *testing.T) {
 	configs := []*AzureAuthConfig{
 		{
@@ -106,6 +132,66 @@ func TestGetServicePrincipalToken(t *testing.T) {
 	assert.Equal(t, token, spt)
 }
 
+func TestGetMultiTenantServicePrincipalToken(t *testing.T) {
+	config := &AzureAuthConfig{
+		TenantID:                      "TenantID",
+		AADClientID:                   "AADClientID",
+		AADClientSecret:               "AADClientSecret",
+		NetworkResourceTenantID:       "NetworkResourceTenantID",
+		NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID",
+	}
+	env := &azure.PublicCloud
+
+	multiTenantToken, err := GetMultiTenantServicePrincipalToken(config, env)
+	assert.NoError(t, err)
+
+	multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{})
+	assert.NoError(t, err)
+
+	spt, err := adal.NewMultiTenantServicePrincipalToken(multiTenantOAuthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint)
+	assert.NoError(t, err)
+
+	assert.Equal(t, multiTenantToken, spt)
+}
+
+func TestGetMultiTenantServicePrincipalTokenNegative(t *testing.T) {
+	env := &azure.PublicCloud
+	for _, config := range CrossTenantNetworkResourceNegativeConfig {
+		_, err := GetMultiTenantServicePrincipalToken(config, env)
+		assert.Error(t, err)
+	}
+}
+
+func TestGetNetworkResourceServicePrincipalToken(t *testing.T) {
+	config := &AzureAuthConfig{
+		TenantID:                      "TenantID",
+		AADClientID:                   "AADClientID",
+		AADClientSecret:               "AADClientSecret",
+		NetworkResourceTenantID:       "NetworkResourceTenantID",
+		NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID",
+	}
+	env := &azure.PublicCloud
+
+	token, err := GetNetworkResourceServicePrincipalToken(config, env)
+	assert.NoError(t, err)
+
+	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil)
+	assert.NoError(t, err)
+
+	spt, err := adal.NewServicePrincipalToken(*oauthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint)
+	assert.NoError(t, err)
+
+	assert.Equal(t, token, spt)
+}
+
+func TestGetNetworkResourceServicePrincipalTokenNegative(t *testing.T) {
+	env := &azure.PublicCloud
+	for _, config := range CrossTenantNetworkResourceNegativeConfig {
+		_, err := GetNetworkResourceServicePrincipalToken(config, env)
+		assert.Error(t, err)
+	}
+}
+
 func TestParseAzureEngironment(t *testing.T) {
 	cases := []struct {
 		cloudName               string

staging/src/k8s.io/legacy-cloud-providers/azure/azure.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/Azure/go-autorest/autorest/azure"
 
 	v1 "k8s.io/api/core/v1"
@@ -476,46 +477,21 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
 		return nil
 	}
 
-	// Initialize Azure clients.
+	// If uses network resources in different AAD Tenant, then prepare corresponding Service Principal Token for VM/VMSS client and network resources client
-	azClientConfig := &azclients.ClientConfig{
+	var multiTenantServicePrincipalToken *adal.MultiTenantServicePrincipalToken
-		Location:                       config.Location,
+	var networkResourceServicePrincipalToken *adal.ServicePrincipalToken
-		SubscriptionID:                 config.SubscriptionID,
+	if az.Config.UsesNetworkResourceInDifferentTenant() {
-		ResourceManagerEndpoint:        env.ResourceManagerEndpoint,
+		multiTenantServicePrincipalToken, err = auth.GetMultiTenantServicePrincipalToken(&az.Config.AzureAuthConfig, &az.Environment)
-		ServicePrincipalToken:          servicePrincipalToken,
+		if err != nil {
-		CloudProviderBackoffRetries:    config.CloudProviderBackoffRetries,
+			return err
-		CloudProviderBackoffDuration:   config.CloudProviderBackoffDuration,
-		ShouldOmitCloudProviderBackoff: config.shouldOmitCloudProviderBackoff(),
-		Backoff:                        &retry.Backoff{Steps: 1},
 		}
-	if config.CloudProviderBackoff {
+		networkResourceServicePrincipalToken, err = auth.GetNetworkResourceServicePrincipalToken(&az.Config.AzureAuthConfig, &az.Environment)
-		azClientConfig.Backoff = &retry.Backoff{
+		if err != nil {
-			Steps:    config.CloudProviderBackoffRetries,
+			return err
-			Factor:   config.CloudProviderBackoffExponent,
-			Duration: time.Duration(config.CloudProviderBackoffDuration) * time.Second,
-			Jitter:   config.CloudProviderBackoffJitter,
 		}
 	}
-	az.RoutesClient = routeclient.New(azClientConfig.WithRateLimiter(config.RouteRateLimit))
-	az.SubnetsClient = subnetclient.New(azClientConfig.WithRateLimiter(config.SubnetsRateLimit))
-	az.InterfacesClient = interfaceclient.New(azClientConfig.WithRateLimiter(config.InterfaceRateLimit))
-	az.RouteTablesClient = routetableclient.New(azClientConfig.WithRateLimiter(config.RouteTableRateLimit))
-	az.LoadBalancerClient = loadbalancerclient.New(azClientConfig.WithRateLimiter(config.LoadBalancerRateLimit))
-	az.SecurityGroupsClient = securitygroupclient.New(azClientConfig.WithRateLimiter(config.SecurityGroupRateLimit))
-	az.PublicIPAddressesClient = publicipclient.New(azClientConfig.WithRateLimiter(config.PublicIPAddressRateLimit))
-	az.VirtualMachineScaleSetsClient = vmssclient.New(azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit))
-	az.VirtualMachineSizesClient = vmsizeclient.New(azClientConfig.WithRateLimiter(config.VirtualMachineSizeRateLimit))
-	az.SnapshotsClient = snapshotclient.New(azClientConfig.WithRateLimiter(config.SnapshotRateLimit))
-	az.StorageAccountClient = storageaccountclient.New(azClientConfig.WithRateLimiter(config.StorageAccountRateLimit))
-	az.VirtualMachinesClient = vmclient.New(azClientConfig.WithRateLimiter(config.VirtualMachineRateLimit))
-	az.DisksClient = diskclient.New(azClientConfig.WithRateLimiter(config.DiskRateLimit))
-	// fileClient is not based on armclient, but it's still backoff retried.
-	az.FileClient = newAzureFileClient(env, azClientConfig.Backoff)
 
-	// Error "not an active Virtual Machine Scale Set VM" is not retriable for VMSS VM.
+	az.configAzureClients(servicePrincipalToken, multiTenantServicePrincipalToken, networkResourceServicePrincipalToken)
-	// But http.StatusNotFound is retriable because of ARM replication latency.
-	vmssVMClientConfig := azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit)
-	vmssVMClientConfig.Backoff = vmssVMClientConfig.Backoff.WithNonRetriableErrors([]string{vmssVMNotActiveErrorMessage}).WithRetriableHTTPStatusCodes([]int{http.StatusNotFound})
-	az.VirtualMachineScaleSetVMsClient = vmssvmclient.New(vmssVMClientConfig)
 
 	if az.MaximumLoadBalancerRuleCount == 0 {
 		az.MaximumLoadBalancerRuleCount = maximumLoadBalancerRuleCount
@@ -561,6 +537,93 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
 	return nil
 }
 
+func (az *Cloud) configAzureClients(
+	servicePrincipalToken *adal.ServicePrincipalToken,
+	multiTenantServicePrincipalToken *adal.MultiTenantServicePrincipalToken,
+	networkResourceServicePrincipalToken *adal.ServicePrincipalToken) {
+	azClientConfig := az.getAzureClientConfig(servicePrincipalToken)
+
+	// Prepare AzureClientConfig for all azure clients
+	interfaceClientConfig := azClientConfig.WithRateLimiter(az.Config.InterfaceRateLimit)
+	vmSizeClientConfig := azClientConfig.WithRateLimiter(az.Config.VirtualMachineSizeRateLimit)
+	snapshotClientConfig := azClientConfig.WithRateLimiter(az.Config.SnapshotRateLimit)
+	storageAccountClientConfig := azClientConfig.WithRateLimiter(az.Config.StorageAccountRateLimit)
+	diskClientConfig := azClientConfig.WithRateLimiter(az.Config.DiskRateLimit)
+	vmClientConfig := azClientConfig.WithRateLimiter(az.Config.VirtualMachineRateLimit)
+	vmssClientConfig := azClientConfig.WithRateLimiter(az.Config.VirtualMachineScaleSetRateLimit)
+	// Error "not an active Virtual Machine Scale Set VM" is not retriable for VMSS VM.
+	// But http.StatusNotFound is retriable because of ARM replication latency.
+	vmssVMClientConfig := azClientConfig.WithRateLimiter(az.Config.VirtualMachineScaleSetRateLimit)
+	vmssVMClientConfig.Backoff = vmssVMClientConfig.Backoff.WithNonRetriableErrors([]string{vmssVMNotActiveErrorMessage}).WithRetriableHTTPStatusCodes([]int{http.StatusNotFound})
+	routeClientConfig := azClientConfig.WithRateLimiter(az.Config.RouteRateLimit)
+	subnetClientConfig := azClientConfig.WithRateLimiter(az.Config.SubnetsRateLimit)
+	routeTableClientConfig := azClientConfig.WithRateLimiter(az.Config.RouteTableRateLimit)
+	loadBalancerClientConfig := azClientConfig.WithRateLimiter(az.Config.LoadBalancerRateLimit)
+	securityGroupClientConfig := azClientConfig.WithRateLimiter(az.Config.SecurityGroupRateLimit)
+	publicIPClientConfig := azClientConfig.WithRateLimiter(az.Config.PublicIPAddressRateLimit)
+
+	// If uses network resources in different AAD Tenant, update Authorizer for VM/VMSS client config
+	if multiTenantServicePrincipalToken != nil {
+		multiTenantServicePrincipalTokenAuthorizer := autorest.NewMultiTenantServicePrincipalTokenAuthorizer(multiTenantServicePrincipalToken)
+		vmClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer
+		vmssClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer
+		vmssVMClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer
+	}
+
+	// If uses network resources in different AAD Tenant, update Authorizer for network resources client config
+	if networkResourceServicePrincipalToken != nil {
+		networkResourceServicePrincipalTokenAuthorizer := autorest.NewBearerAuthorizer(networkResourceServicePrincipalToken)
+		routeClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
+		subnetClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
+		routeTableClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
+		loadBalancerClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
+		securityGroupClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
+		publicIPClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
+	}
+
+	// Initialize all azure clients based on client config
+	az.InterfacesClient = interfaceclient.New(interfaceClientConfig)
+	az.VirtualMachineSizesClient = vmsizeclient.New(vmSizeClientConfig)
+	az.SnapshotsClient = snapshotclient.New(snapshotClientConfig)
+	az.StorageAccountClient = storageaccountclient.New(storageAccountClientConfig)
+	az.DisksClient = diskclient.New(diskClientConfig)
+	az.VirtualMachinesClient = vmclient.New(vmClientConfig)
+	az.VirtualMachineScaleSetsClient = vmssclient.New(vmssClientConfig)
+	az.VirtualMachineScaleSetVMsClient = vmssvmclient.New(vmssVMClientConfig)
+	az.RoutesClient = routeclient.New(routeClientConfig)
+	az.SubnetsClient = subnetclient.New(subnetClientConfig)
+	az.RouteTablesClient = routetableclient.New(routeTableClientConfig)
+	az.LoadBalancerClient = loadbalancerclient.New(loadBalancerClientConfig)
+	az.SecurityGroupsClient = securitygroupclient.New(securityGroupClientConfig)
+	az.PublicIPAddressesClient = publicipclient.New(publicIPClientConfig)
+	// fileClient is not based on armclient, but it's still backoff retried.
+	az.FileClient = newAzureFileClient(&az.Environment, azClientConfig.Backoff)
+}
+
+func (az *Cloud) getAzureClientConfig(servicePrincipalToken *adal.ServicePrincipalToken) *azclients.ClientConfig {
+	azClientConfig := &azclients.ClientConfig{
+		Location:                       az.Config.Location,
+		SubscriptionID:                 az.Config.SubscriptionID,
+		ResourceManagerEndpoint:        az.Environment.ResourceManagerEndpoint,
+		CloudProviderBackoffRetries:    az.Config.CloudProviderBackoffRetries,
+		CloudProviderBackoffDuration:   az.Config.CloudProviderBackoffDuration,
+		ShouldOmitCloudProviderBackoff: az.Config.shouldOmitCloudProviderBackoff(),
+		Authorizer:                     autorest.NewBearerAuthorizer(servicePrincipalToken),
+		Backoff:                        &retry.Backoff{Steps: 1},
+	}
+
+	if az.Config.CloudProviderBackoff {
+		azClientConfig.Backoff = &retry.Backoff{
+			Steps:    az.Config.CloudProviderBackoffRetries,
+			Factor:   az.Config.CloudProviderBackoffExponent,
+			Duration: time.Duration(az.Config.CloudProviderBackoffDuration) * time.Second,
+			Jitter:   az.Config.CloudProviderBackoffJitter,
+		}
+	}
+
+	return azClientConfig
+}
+
 // parseConfig returns a parsed configuration for an Azure cloudprovider config file
 func parseConfig(configReader io.Reader) (*Config, error) {
 	var config Config

staging/src/k8s.io/legacy-cloud-providers/azure/azure_ratelimit_test.go

@@ -54,6 +54,8 @@ var (
 		"loadBalancerRateLimit": {
 			"cloudProviderRatelimit": false,
 		},
+		"networkResourceTenantId": "networkResourceTenantId",
+		"networkResourceSubscriptionId": "networkResourceSubscriptionId",
 		"availabilitySetNodesCacheTTLInSeconds": 100,
 		"vmssCacheTTLInSeconds": 100,
 		"vmssVirtualMachinesCacheTTLInSeconds": 100,
@@ -100,6 +102,8 @@ func TestParseConfig(t *testing.T) {
 			SubscriptionID:                "subscriptionId",
 			TenantID:                      "tenantId",
 			UseManagedIdentityExtension:   true,
+			NetworkResourceTenantID:       "networkResourceTenantId",
+			NetworkResourceSubscriptionID: "networkResourceSubscriptionId",
 		},
 		CloudProviderBackoff:         true,
 		CloudProviderBackoffDuration: 1,

staging/src/k8s.io/legacy-cloud-providers/azure/clients/BUILD

@@ -12,7 +12,7 @@ go_library(
     deps = [
         "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library",
-        "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
     ],
 )
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/azure_client_config.go

@@ -19,7 +19,7 @@ limitations under the License.
 package clients
 
 import (
-	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/Azure/go-autorest/autorest"
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/legacy-cloud-providers/azure/retry"
 )
@@ -29,7 +29,7 @@ type ClientConfig struct {
 	Location                string
 	SubscriptionID          string
 	ResourceManagerEndpoint string
-	ServicePrincipalToken   *adal.ServicePrincipalToken
+	Authorizer              autorest.Authorizer
 	RateLimitConfig         *RateLimitConfig
 	Backoff                 *retry.Backoff
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/diskclient/azure_diskclient.go

@@ -54,7 +54,7 @@ type Client struct {
 // New creates a new Disk client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient.go

@@ -55,7 +55,7 @@ type Client struct {
 // New creates a new network interface client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient/azure_loadbalancerclient.go

@@ -56,7 +56,7 @@ type Client struct {
 // New creates a new LoadBalancer client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/publicipclient/azure_publicipclient.go

@@ -56,7 +56,7 @@ type Client struct {
 // New creates a new PublicIPAddress client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/routeclient/azure_routeclient.go

@@ -54,7 +54,7 @@ type Client struct {
 // New creates a new Route client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/routetableclient/azure_routetableclient.go

@@ -54,7 +54,7 @@ type Client struct {
 // New creates a new RouteTable client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient/azure_securitygroupclient.go

@@ -56,7 +56,7 @@ type Client struct {
 // New creates a new SecurityGroup client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/azure_snapshotclient.go

@@ -56,7 +56,7 @@ type Client struct {
 // New creates a new Snapshot client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/storageaccountclient/azure_storageaccountclient.go

@@ -56,7 +56,7 @@ type Client struct {
 // New creates a new StorageAccount client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/azure_subnetclient.go

@@ -55,7 +55,7 @@ type Client struct {
 // New creates a new Subnet client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient.go

@@ -56,7 +56,7 @@ type Client struct {
 // New creates a new VirtualMachine client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/azure_vmsizeclient.go

@@ -55,7 +55,7 @@ type Client struct {
 // New creates a new VirtualMachineSize client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient.go

@@ -56,7 +56,7 @@ type Client struct {
 // New creates a new VMSS client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/azure_vmssvmclient.go

@@ -56,7 +56,7 @@ type Client struct {
 // New creates a new vmssVM client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
-	authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken)
+	authorizer := config.Authorizer
 	armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff)
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)