Use new clients in Azure cloud provider

staging/src/k8s.io/legacy-cloud-providers/azure/BUILD

@@ -66,6 +66,14 @@ go_library(
         "//staging/src/k8s.io/component-base/featuregate:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/auth:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/publicipclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/routeclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/routetableclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/metrics:go_default_library",

staging/src/k8s.io/legacy-cloud-providers/azure/azure.go

@@ -46,6 +46,14 @@ import (
 	"k8s.io/klog"
 	"k8s.io/legacy-cloud-providers/azure/auth"
 	azclients "k8s.io/legacy-cloud-providers/azure/clients"
+	"k8s.io/legacy-cloud-providers/azure/clients/interfaceclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/publicipclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/routeclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/routetableclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/subnetclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/vmclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/vmssclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient"
 	"k8s.io/legacy-cloud-providers/azure/retry"
@@ -476,24 +484,27 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
 			Jitter:   config.CloudProviderBackoffJitter,
 		}
 	}
-	az.DisksClient = newAzDisksClient(azClientConfig.WithRateLimiter(config.DiskRateLimit))
+	az.RoutesClient = routeclient.New(azClientConfig.WithRateLimiter(config.RouteRateLimit))
-	az.SnapshotsClient = newSnapshotsClient(azClientConfig.WithRateLimiter(config.SnapshotRateLimit))
+	az.SubnetsClient = subnetclient.New(azClientConfig.WithRateLimiter(config.SubnetsRateLimit))
-	az.RoutesClient = newAzRoutesClient(azClientConfig.WithRateLimiter(config.RouteRateLimit))
+	az.InterfacesClient = interfaceclient.New(azClientConfig.WithRateLimiter(config.InterfaceRateLimit))
-	az.SubnetsClient = newAzSubnetsClient(azClientConfig.WithRateLimiter(config.SubnetsRateLimit))
+	az.RouteTablesClient = routetableclient.New(azClientConfig.WithRateLimiter(config.RouteTableRateLimit))
-	az.InterfacesClient = newAzInterfacesClient(azClientConfig.WithRateLimiter(config.InterfaceRateLimit))
+	az.LoadBalancerClient = loadbalancerclient.New(azClientConfig.WithRateLimiter(config.LoadBalancerRateLimit))
-	az.RouteTablesClient = newAzRouteTablesClient(azClientConfig.WithRateLimiter(config.RouteTableRateLimit))
+	az.SecurityGroupsClient = securitygroupclient.New(azClientConfig.WithRateLimiter(config.SecurityGroupRateLimit))
-	az.LoadBalancerClient = newAzLoadBalancersClient(azClientConfig.WithRateLimiter(config.LoadBalancerRateLimit))
+	az.VirtualMachinesClient = vmclient.New(azClientConfig.WithRateLimiter(config.VirtualMachineRateLimit))
-	az.SecurityGroupsClient = newAzSecurityGroupsClient(azClientConfig.WithRateLimiter(config.SecurityGroupRateLimit))
+	az.PublicIPAddressesClient = publicipclient.New(azClientConfig.WithRateLimiter(config.PublicIPAddressRateLimit))
-	az.StorageAccountClient = newAzStorageAccountClient(azClientConfig.WithRateLimiter(config.StorageAccountRateLimit))
-	az.VirtualMachinesClient = newAzVirtualMachinesClient(azClientConfig.WithRateLimiter(config.VirtualMachineRateLimit))
-	az.PublicIPAddressesClient = newAzPublicIPAddressesClient(azClientConfig.WithRateLimiter(config.PublicIPAddressRateLimit))
-	az.VirtualMachineSizesClient = newAzVirtualMachineSizesClient(azClientConfig.WithRateLimiter(config.VirtualMachineSizeRateLimit))
-
 	az.VirtualMachineScaleSetsClient = vmssclient.New(azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit))
+
+	// Error "not an active Virtual Machine Scale Set VM" is not retriable for VMSS VM.
 	vmssVMClientConfig := azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit)
 	vmssVMClientConfig.Backoff = vmssVMClientConfig.Backoff.WithNonRetriableErrors([]string{vmssVMNotActiveErrorMessage})
 	az.VirtualMachineScaleSetVMsClient = vmssvmclient.New(vmssVMClientConfig)
 
+	// TODO(feiskyer): refactor the following clients to use armclient
+	az.DisksClient = newAzDisksClient(azClientConfig.WithRateLimiter(config.DiskRateLimit))
+	az.SnapshotsClient = newSnapshotsClient(azClientConfig.WithRateLimiter(config.SnapshotRateLimit))
+	az.StorageAccountClient = newAzStorageAccountClient(azClientConfig.WithRateLimiter(config.StorageAccountRateLimit))
+	az.VirtualMachineSizesClient = newAzVirtualMachineSizesClient(azClientConfig.WithRateLimiter(config.VirtualMachineSizeRateLimit))
+
 	// TODO(feiskyer): refactor azureFileClient to Interface.
 	az.FileClient = &azureFileClient{env: *env}
 

staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go

@@ -86,33 +86,8 @@ func (az *Cloud) GetVirtualMachineWithRetry(name types.NodeName, crt cacheReadTy
 	return machine, err
 }
 
-// ListVirtualMachinesWithRetry invokes az.VirtualMachinesClient.List with exponential backoff retry
-func (az *Cloud) ListVirtualMachinesWithRetry(resourceGroup string) ([]compute.VirtualMachine, error) {
-	allNodes := []compute.VirtualMachine{}
-	err := wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		var retryErr *retry.Error
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-		allNodes, retryErr = az.VirtualMachinesClient.List(ctx, resourceGroup)
-		if retryErr != nil {
-			klog.Errorf("VirtualMachinesClient.List(%v) - backoff: failure, will retry,err=%v",
-				resourceGroup,
-				retryErr)
-			return false, retryErr.Error()
-		}
-		klog.V(2).Infof("VirtualMachinesClient.List(%v) - backoff: success", resourceGroup)
-		return true, nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return allNodes, nil
-}
-
 // ListVirtualMachines invokes az.VirtualMachinesClient.List with exponential backoff retry
 func (az *Cloud) ListVirtualMachines(resourceGroup string) ([]compute.VirtualMachine, error) {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -123,9 +98,6 @@ func (az *Cloud) ListVirtualMachines(resourceGroup string) ([]compute.VirtualMac
 	}
 	klog.V(2).Infof("VirtualMachinesClient.List(%v) success", resourceGroup)
 	return allNodes, nil
-	}
-
-	return az.ListVirtualMachinesWithRetry(resourceGroup)
 }
 
 // getPrivateIPsForMachine is wrapper for optional backoff getting private ips
@@ -183,7 +155,6 @@ func (az *Cloud) GetIPForMachineWithRetry(name types.NodeName) (string, string,
 
 // CreateOrUpdateSecurityGroup invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry
 func (az *Cloud) CreateOrUpdateSecurityGroup(service *v1.Service, sg network.SecurityGroup) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -197,53 +168,21 @@ func (az *Cloud) CreateOrUpdateSecurityGroup(service *v1.Service, sg network.Sec
 
 	// Invalidate the cache because ETAG precondition mismatch.
 	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
+		klog.V(3).Infof("SecurityGroup cache for %s is cleanup because of http.StatusPreconditionFailed", *sg.Name)
 		az.nsgCache.Delete(*sg.Name)
 	}
 
 	// Invalidate the cache because another new operation has canceled the current request.
 	if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) {
+		klog.V(3).Infof("SecurityGroup cache for %s is cleanup because CreateOrUpdateSecurityGroup is canceld by another operation", *sg.Name)
 		az.nsgCache.Delete(*sg.Name)
 	}
 
 	return rerr.Error()
-	}
-
-	return az.CreateOrUpdateSGWithRetry(service, sg)
-}
-
-// CreateOrUpdateSGWithRetry invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry
-func (az *Cloud) CreateOrUpdateSGWithRetry(service *v1.Service, sg network.SecurityGroup) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rerr := az.SecurityGroupsClient.CreateOrUpdate(ctx, az.SecurityGroupResourceGroup, *sg.Name, sg, to.String(sg.Etag))
-		klog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%s): end", *sg.Name)
-		if rerr == nil {
-			// Invalidate the cache right after updating
-			az.nsgCache.Delete(*sg.Name)
-			return true, nil
-		}
-
-		// Invalidate the cache and abort backoff because ETAG precondition mismatch.
-		if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
-			az.nsgCache.Delete(*sg.Name)
-			return true, rerr.Error()
-		}
-
-		// Invalidate the cache and abort backoff because another new operation has canceled the current request.
-		if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) {
-			az.nsgCache.Delete(*sg.Name)
-			return true, rerr.Error()
-		}
-
-		return !rerr.Retriable, rerr.Error()
-	})
 }
 
 // CreateOrUpdateLB invokes az.LoadBalancerClient.CreateOrUpdate with exponential backoff retry
 func (az *Cloud) CreateOrUpdateLB(service *v1.Service, lb network.LoadBalancer) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -258,50 +197,20 @@ func (az *Cloud) CreateOrUpdateLB(service *v1.Service, lb network.LoadBalancer)
 
 	// Invalidate the cache because ETAG precondition mismatch.
 	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
+		klog.V(3).Infof("LoadBalancer cache for %s is cleanup because of http.StatusPreconditionFailed", *lb.Name)
 		az.lbCache.Delete(*lb.Name)
 	}
 	// Invalidate the cache because another new operation has canceled the current request.
 	if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) {
+		klog.V(3).Infof("LoadBalancer cache for %s is cleanup because CreateOrUpdate is canceled by another operation", *lb.Name)
 		az.lbCache.Delete(*lb.Name)
 	}
+
 	return rerr.Error()
-	}
-
-	return az.createOrUpdateLBWithRetry(service, lb)
-}
-
-// createOrUpdateLBWithRetry invokes az.LoadBalancerClient.CreateOrUpdate with exponential backoff retry
-func (az *Cloud) createOrUpdateLBWithRetry(service *v1.Service, lb network.LoadBalancer) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rgName := az.getLoadBalancerResourceGroup()
-		rerr := az.LoadBalancerClient.CreateOrUpdate(ctx, rgName, *lb.Name, lb, to.String(lb.Etag))
-		klog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%s): end", *lb.Name)
-		if rerr == nil {
-			// Invalidate the cache right after updating
-			az.lbCache.Delete(*lb.Name)
-			return true, nil
-		}
-
-		// Invalidate the cache and abort backoff because ETAG precondition mismatch.
-		if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
-			az.lbCache.Delete(*lb.Name)
-			return true, rerr.Error()
-		}
-		// Invalidate the cache and abort backoff because another new operation has canceled the current request.
-		if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) {
-			az.lbCache.Delete(*lb.Name)
-			return true, rerr.Error()
-		}
-		return !rerr.Retriable, rerr.Error()
-	})
 }
 
 // ListLB invokes az.LoadBalancerClient.List with exponential backoff retry
 func (az *Cloud) ListLB(service *v1.Service) ([]network.LoadBalancer, error) {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -313,43 +222,11 @@ func (az *Cloud) ListLB(service *v1.Service) ([]network.LoadBalancer, error) {
 		return nil, rerr.Error()
 	}
 	klog.V(2).Infof("LoadBalancerClient.List(%v) success", rgName)
-		return allLBs, nil
-	}
-
-	return az.listLBWithRetry(service)
-}
-
-// listLBWithRetry invokes az.LoadBalancerClient.List with exponential backoff retry
-func (az *Cloud) listLBWithRetry(service *v1.Service) ([]network.LoadBalancer, error) {
-	var retryErr *retry.Error
-	var allLBs []network.LoadBalancer
-
-	err := wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rgName := az.getLoadBalancerResourceGroup()
-		allLBs, retryErr = az.LoadBalancerClient.List(ctx, rgName)
-		if retryErr != nil {
-			az.Event(service, v1.EventTypeWarning, "ListLoadBalancers", retryErr.Error().Error())
-			klog.Errorf("LoadBalancerClient.List(%v) - backoff: failure, will retry,err=%v",
-				rgName,
-				retryErr)
-			return false, retryErr.Error()
-		}
-		klog.V(2).Infof("LoadBalancerClient.List(%v) - backoff: success", az.ResourceGroup)
-		return true, nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
 	return allLBs, nil
 }
 
 // ListPIP list the PIP resources in the given resource group
 func (az *Cloud) ListPIP(service *v1.Service, pipResourceGroup string) ([]network.PublicIPAddress, error) {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -359,43 +236,13 @@ func (az *Cloud) ListPIP(service *v1.Service, pipResourceGroup string) ([]networ
 		klog.Errorf("PublicIPAddressesClient.List(%v) failure with err=%v", pipResourceGroup, rerr)
 		return nil, rerr.Error()
 	}
+
 	klog.V(2).Infof("PublicIPAddressesClient.List(%v) success", pipResourceGroup)
-		return allPIPs, nil
-	}
-
-	return az.listPIPWithRetry(service, pipResourceGroup)
-}
-
-// listPIPWithRetry list the PIP resources in the given resource group
-func (az *Cloud) listPIPWithRetry(service *v1.Service, pipResourceGroup string) ([]network.PublicIPAddress, error) {
-	var allPIPs []network.PublicIPAddress
-
-	err := wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		var retryErr *retry.Error
-		allPIPs, retryErr = az.PublicIPAddressesClient.List(ctx, pipResourceGroup)
-		if retryErr != nil {
-			az.Event(service, v1.EventTypeWarning, "ListPublicIPs", retryErr.Error().Error())
-			klog.Errorf("PublicIPAddressesClient.List(%v) - backoff: failure, will retry,err=%v",
-				pipResourceGroup,
-				retryErr)
-			return false, retryErr.Error()
-		}
-		klog.V(2).Infof("PublicIPAddressesClient.List(%v) - backoff: success", pipResourceGroup)
-		return true, nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
 	return allPIPs, nil
 }
 
 // CreateOrUpdatePIP invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
 func (az *Cloud) CreateOrUpdatePIP(service *v1.Service, pipResourceGroup string, pip network.PublicIPAddress) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -408,32 +255,10 @@ func (az *Cloud) CreateOrUpdatePIP(service *v1.Service, pipResourceGroup string,
 	}
 
 	return nil
-	}
-
-	return az.createOrUpdatePIPWithRetry(service, pipResourceGroup, pip)
-}
-
-// createOrUpdatePIPWithRetry invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
-func (az *Cloud) createOrUpdatePIPWithRetry(service *v1.Service, pipResourceGroup string, pip network.PublicIPAddress) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rerr := az.PublicIPAddressesClient.CreateOrUpdate(ctx, pipResourceGroup, *pip.Name, pip)
-		klog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): end", pipResourceGroup, *pip.Name)
-		if rerr != nil {
-			klog.Errorf("PublicIPAddressesClient.CreateOrUpdate(%s, %s) failed: %s", pipResourceGroup, *pip.Name, rerr.Error().Error())
-			az.Event(service, v1.EventTypeWarning, "CreateOrUpdatePublicIPAddress", rerr.Error().Error())
-			return !rerr.Retriable, rerr.Error()
-		}
-
-		return true, nil
-	})
 }
 
 // CreateOrUpdateInterface invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
 func (az *Cloud) CreateOrUpdateInterface(service *v1.Service, nic network.Interface) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -446,32 +271,10 @@ func (az *Cloud) CreateOrUpdateInterface(service *v1.Service, nic network.Interf
 	}
 
 	return nil
-	}
-
-	return az.createOrUpdateInterfaceWithRetry(service, nic)
-}
-
-// createOrUpdateInterfaceWithRetry invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
-func (az *Cloud) createOrUpdateInterfaceWithRetry(service *v1.Service, nic network.Interface) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rerr := az.InterfacesClient.CreateOrUpdate(ctx, az.ResourceGroup, *nic.Name, nic)
-		klog.V(10).Infof("InterfacesClient.CreateOrUpdate(%s): end", *nic.Name)
-		if rerr != nil {
-			klog.Errorf("InterfacesClient.CreateOrUpdate(%s) faild: %s", *nic.Name, rerr.Error().Error())
-			az.Event(service, v1.EventTypeWarning, "CreateOrUpdateInterface", rerr.Error().Error())
-			return !rerr.Retriable, rerr.Error()
-		}
-
-		return true, nil
-	})
 }
 
 // DeletePublicIP invokes az.PublicIPAddressesClient.Delete with exponential backoff retry
 func (az *Cloud) DeletePublicIP(service *v1.Service, pipResourceGroup string, pipName string) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -483,31 +286,10 @@ func (az *Cloud) DeletePublicIP(service *v1.Service, pipResourceGroup string, pi
 	}
 
 	return nil
-	}
-
-	return az.deletePublicIPWithRetry(service, pipResourceGroup, pipName)
-}
-
-// deletePublicIPWithRetry invokes az.PublicIPAddressesClient.Delete with exponential backoff retry
-func (az *Cloud) deletePublicIPWithRetry(service *v1.Service, pipResourceGroup string, pipName string) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rerr := az.PublicIPAddressesClient.Delete(ctx, pipResourceGroup, pipName)
-		if rerr != nil {
-			klog.Errorf("PublicIPAddressesClient.Delete(%s) failed: %s", pipName, rerr.Error().Error())
-			az.Event(service, v1.EventTypeWarning, "DeletePublicIPAddress", rerr.Error().Error())
-			return !rerr.Retriable, rerr.Error()
-		}
-
-		return true, nil
-	})
 }
 
 // DeleteLB invokes az.LoadBalancerClient.Delete with exponential backoff retry
 func (az *Cloud) DeleteLB(service *v1.Service, lbName string) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -522,34 +304,10 @@ func (az *Cloud) DeleteLB(service *v1.Service, lbName string) error {
 	klog.Errorf("LoadBalancerClient.Delete(%s) failed: %s", lbName, rerr.Error().Error())
 	az.Event(service, v1.EventTypeWarning, "DeleteLoadBalancer", rerr.Error().Error())
 	return rerr.Error()
-	}
-
-	return az.deleteLBWithRetry(service, lbName)
-}
-
-// deleteLBWithRetry invokes az.LoadBalancerClient.Delete with exponential backoff retry
-func (az *Cloud) deleteLBWithRetry(service *v1.Service, lbName string) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rgName := az.getLoadBalancerResourceGroup()
-		rerr := az.LoadBalancerClient.Delete(ctx, rgName, lbName)
-		if rerr == nil {
-			// Invalidate the cache right after deleting
-			az.lbCache.Delete(lbName)
-			return true, nil
-		}
-
-		klog.Errorf("LoadBalancerClient.Delete(%s) failed: %s", lbName, rerr.Error().Error())
-		az.Event(service, v1.EventTypeWarning, "CreateOrUpdateInterface", rerr.Error().Error())
-		return !rerr.Retriable, rerr.Error()
-	})
 }
 
 // CreateOrUpdateRouteTable invokes az.RouteTablesClient.CreateOrUpdate with exponential backoff retry
 func (az *Cloud) CreateOrUpdateRouteTable(routeTable network.RouteTable) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -562,49 +320,20 @@ func (az *Cloud) CreateOrUpdateRouteTable(routeTable network.RouteTable) error {
 
 	// Invalidate the cache because etag mismatch.
 	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
+		klog.V(3).Infof("Route table cache for %s is cleanup because of http.StatusPreconditionFailed", *routeTable.Name)
 		az.rtCache.Delete(*routeTable.Name)
 	}
 	// Invalidate the cache because another new operation has canceled the current request.
 	if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) {
+		klog.V(3).Infof("Route table cache for %s is cleanup because CreateOrUpdateRouteTable is canceld by another operation", *routeTable.Name)
 		az.rtCache.Delete(*routeTable.Name)
 	}
 	klog.Errorf("RouteTablesClient.CreateOrUpdate(%s) failed: %v", az.RouteTableName, rerr.Error())
 	return rerr.Error()
-	}
-
-	return az.createOrUpdateRouteTableWithRetry(routeTable)
-}
-
-// createOrUpdateRouteTableWithRetry invokes az.RouteTablesClient.CreateOrUpdate with exponential backoff retry
-func (az *Cloud) createOrUpdateRouteTableWithRetry(routeTable network.RouteTable) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rerr := az.RouteTablesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, routeTable, to.String(routeTable.Etag))
-		if rerr == nil {
-			az.rtCache.Delete(*routeTable.Name)
-			return true, nil
-		}
-
-		// Invalidate the cache and abort backoff because ETAG precondition mismatch.
-		if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
-			az.rtCache.Delete(*routeTable.Name)
-			return true, rerr.Error()
-		}
-		// Invalidate the cache and abort backoff because another new operation has canceled the current request.
-		if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) {
-			az.rtCache.Delete(*routeTable.Name)
-			return true, rerr.Error()
-		}
-		klog.Errorf("RouteTablesClient.CreateOrUpdate(%s) failed: %v", az.RouteTableName, rerr.Error())
-		return !rerr.Retriable, rerr.Error()
-	})
 }
 
 // CreateOrUpdateRoute invokes az.RoutesClient.CreateOrUpdate with exponential backoff retry
 func (az *Cloud) CreateOrUpdateRoute(route network.Route) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -616,50 +345,19 @@ func (az *Cloud) CreateOrUpdateRoute(route network.Route) error {
 	}
 
 	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
+		klog.V(3).Infof("Route cache for %s is cleanup because of http.StatusPreconditionFailed", *route.Name)
 		az.rtCache.Delete(az.RouteTableName)
 	}
 	// Invalidate the cache because another new operation has canceled the current request.
 	if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) {
+		klog.V(3).Infof("Route cache for %s is cleanup because CreateOrUpdateRouteTable is canceld by another operation", *route.Name)
 		az.rtCache.Delete(az.RouteTableName)
 	}
 	return rerr.Error()
-	}
-
-	return az.createOrUpdateRouteWithRetry(route)
-}
-
-// createOrUpdateRouteWithRetry invokes az.RoutesClient.CreateOrUpdate with exponential backoff retry
-func (az *Cloud) createOrUpdateRouteWithRetry(route network.Route) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rerr := az.RoutesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, *route.Name, route, to.String(route.Etag))
-		klog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): end", *route.Name)
-		if rerr == nil {
-			az.rtCache.Delete(az.RouteTableName)
-			return true, nil
-		}
-
-		// Invalidate the cache and abort backoff because ETAG precondition mismatch.
-		if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
-			az.rtCache.Delete(az.RouteTableName)
-			return true, rerr.Error()
-		}
-
-		// Invalidate the cache and abort backoff because another new operation has canceled the current request.
-		if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) {
-			az.rtCache.Delete(az.RouteTableName)
-			return true, rerr.Error()
-		}
-
-		return !rerr.Retriable, rerr.Error()
-	})
 }
 
 // DeleteRouteWithName invokes az.RoutesClient.CreateOrUpdate with exponential backoff retry
 func (az *Cloud) DeleteRouteWithName(routeName string) error {
-	if az.Config.shouldOmitCloudProviderBackoff() {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
@@ -671,26 +369,6 @@ func (az *Cloud) DeleteRouteWithName(routeName string) error {
 
 	klog.Errorf("RoutesClient.Delete(%s, %s) failed: %v", az.RouteTableName, routeName, rerr.Error())
 	return rerr.Error()
-	}
-
-	return az.deleteRouteWithRetry(routeName)
-}
-
-// deleteRouteWithRetry invokes az.RoutesClient.Delete with exponential backoff retry
-func (az *Cloud) deleteRouteWithRetry(routeName string) error {
-	return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
-		ctx, cancel := getContextWithCancel()
-		defer cancel()
-
-		rerr := az.RoutesClient.Delete(ctx, az.RouteTableResourceGroup, az.RouteTableName, routeName)
-		klog.V(10).Infof("RoutesClient.Delete(%s,%s): end", az.RouteTableName, routeName)
-		if rerr == nil {
-			return true, nil
-		}
-
-		klog.Errorf("RoutesClient.Delete(%s, %s) failed: %v", az.RouteTableName, routeName, rerr.Error())
-		return !rerr.Retriable, rerr.Error()
-	})
 }
 
 // CreateOrUpdateVMSS invokes az.VirtualMachineScaleSetsClient.Update().

staging/src/k8s.io/legacy-cloud-providers/azure/clients/BUILD

@@ -28,6 +28,14 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:all-srcs",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient:all-srcs",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient:all-srcs",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/publicipclient:all-srcs",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/routeclient:all-srcs",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/routetableclient:all-srcs",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient:all-srcs",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient:all-srcs",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient:all-srcs",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient:all-srcs",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient:all-srcs",
     ],

staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go

@@ -311,6 +311,21 @@ func (c *Client) GetResource(ctx context.Context, resourceID, expand string) (*h
 	return c.Send(ctx, request)
 }
 
+// GetResourceWithDecorators get a resource with decorators by resource ID
+func (c *Client) GetResourceWithDecorators(ctx context.Context, resourceID string, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
+	getDecorators := []autorest.PrepareDecorator{
+		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
+	}
+	getDecorators = append(getDecorators, decorators...)
+	request, err := c.PrepareGetRequest(ctx, getDecorators...)
+	if err != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "get.prepare", resourceID, err)
+		return nil, retry.NewError(false, err)
+	}
+
+	return c.Send(ctx, request)
+}
+
 // PutResource puts a resource by resource ID
 func (c *Client) PutResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) {
 	putDecorators := []autorest.PrepareDecorator{

staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/interface.go

@@ -76,6 +76,9 @@ type Interface interface {
 	// GetResource get a resource by resource ID
 	GetResource(ctx context.Context, resourceID, expand string) (*http.Response, *retry.Error)
 
+	//GetResourceWithDecorators get a resource with decorators by resource ID
+	GetResourceWithDecorators(ctx context.Context, resourceID string, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error)
+
 	// PostResource posts a resource by resource ID
 	PostResource(ctx context.Context, resourceID, action string, parameters interface{}) (*http.Response, *retry.Error)
 

staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient/interface.go

@@ -302,6 +302,21 @@ func (mr *MockInterfaceMockRecorder) GetResource(ctx, resourceID, expand interfa
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockInterface)(nil).GetResource), ctx, resourceID, expand)
 }
 
+// GetResourceWithDecorators mocks base method
+func (m *MockInterface) GetResourceWithDecorators(ctx context.Context, resourceID string, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetResourceWithDecorators", ctx, resourceID, decorators)
+	ret0, _ := ret[0].(*http.Response)
+	ret1, _ := ret[1].(*retry.Error)
+	return ret0, ret1
+}
+
+// GetResourceWithDecorators indicates an expected call of GetResourceWithDecorators
+func (mr *MockInterfaceMockRecorder) GetResourceWithDecorators(ctx, resourceID, decorators interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResourceWithDecorators", reflect.TypeOf((*MockInterface)(nil).GetResourceWithDecorators), ctx, resourceID, decorators)
+}
+
 // PostResource mocks base method
 func (m *MockInterface) PostResource(ctx context.Context, resourceID, action string, parameters interface{}) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()

staging/src/k8s.io/legacy-cloud-providers/azure/clients/routeclient/BUILD

@@ -32,7 +32,6 @@ go_test(
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network:go_default_library",
-        "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/golang/mock/gomock:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",

staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/BUILD

@@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
-        "azure_vmssclientvm.go",
+        "azure_vmssvmclient.go",
         "doc.go",
         "interface.go",
     ],

vendor/modules.txt

@@ -1841,6 +1841,14 @@ k8s.io/legacy-cloud-providers/azure
 k8s.io/legacy-cloud-providers/azure/auth
 k8s.io/legacy-cloud-providers/azure/clients
 k8s.io/legacy-cloud-providers/azure/clients/armclient
+k8s.io/legacy-cloud-providers/azure/clients/interfaceclient
+k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient
+k8s.io/legacy-cloud-providers/azure/clients/publicipclient
+k8s.io/legacy-cloud-providers/azure/clients/routeclient
+k8s.io/legacy-cloud-providers/azure/clients/routetableclient
+k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient
+k8s.io/legacy-cloud-providers/azure/clients/subnetclient
+k8s.io/legacy-cloud-providers/azure/clients/vmclient
 k8s.io/legacy-cloud-providers/azure/clients/vmssclient
 k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient
 k8s.io/legacy-cloud-providers/azure/metrics