Use new clients for vmss cache

pkg/cloudprovider/providers/azure/azure_backoff.go

@@ -17,13 +17,17 @@ limitations under the License.
 package azure
 
 import (
-	"k8s.io/apimachinery/pkg/util/wait"
+	"context"
+	"net/http"
 
 	"github.com/Azure/azure-sdk-for-go/arm/compute"
 	"github.com/Azure/azure-sdk-for-go/arm/network"
+	computepreview "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/golang/glog"
+
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // requestBackoff if backoff is disabled in cloud provider it
@@ -345,6 +349,15 @@ func (az *Cloud) CreateOrUpdateVMWithRetry(vmName string, newVM compute.VirtualM
 	})
 }
 
+// UpdateVmssVMWithRetry invokes az.VirtualMachineScaleSetVMsClient.Update with exponential backoff retry
+func (az *Cloud) UpdateVmssVMWithRetry(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, parameters computepreview.VirtualMachineScaleSetVM) error {
+	return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
+		resp, err := az.VirtualMachineScaleSetVMsClient.Update(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters)
+		glog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s,%s): end", VMScaleSetName, instanceID)
+		return processHTTPRetryResponse(resp, err)
+	})
+}
+
 // A wait.ConditionFunc function to deal with common HTTP backoff response conditions
 func processRetryResponse(resp autorest.Response, err error) (bool, error) {
 	if isSuccessHTTPResponse(resp) {
@@ -380,3 +393,36 @@ func isSuccessHTTPResponse(resp autorest.Response) bool {
 	}
 	return false
 }
+
+func shouldRetryHTTPRequest(resp *http.Response, err error) bool {
+	if err != nil {
+		return true
+	}
+
+	if resp != nil {
+		// HTTP 4xx or 5xx suggests we should retry
+		if 399 < resp.StatusCode && resp.StatusCode < 600 {
+			return true
+		}
+	}
+
+	return false
+}
+
+func processHTTPRetryResponse(resp *http.Response, err error) (bool, error) {
+	if resp != nil {
+		// HTTP 2xx suggests a successful response
+		if 199 < resp.StatusCode && resp.StatusCode < 300 {
+			return true, nil
+		}
+	}
+
+	if shouldRetryHTTPRequest(resp, err) {
+		glog.Errorf("backoff: failure, will retry, HTTP response=%d, err=%v", resp.StatusCode, err)
+		// suppress the error object so that backoff process continues
+		return false, nil
+	}
+
+	// Fall-through: stop periodic backoff
+	return true, nil
+}

pkg/cloudprovider/providers/azure/azure_client.go

@@ -140,6 +140,10 @@ type azVirtualMachinesClient struct {
 	rateLimiter flowcontrol.RateLimiter
 }
 
+func getContextWithCancel() (context.Context, context.CancelFunc) {
+	return context.WithCancel(context.Background())
+}
+
 func newAzVirtualMachinesClient(config *azClientConfig) *azVirtualMachinesClient {
 	virtualMachinesClient := compute.NewVirtualMachinesClient(config.subscriptionID)
 	virtualMachinesClient.BaseURI = config.resourceManagerEndpoint

pkg/cloudprovider/providers/azure/azure_vmss.go

@@ -24,8 +24,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/Azure/azure-sdk-for-go/arm/compute"
 	"github.com/Azure/azure-sdk-for-go/arm/network"
+	computepreview "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
 	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/golang/glog"
 
@@ -89,7 +89,7 @@ func newScaleSet(az *Cloud) (VMSet, error) {
 
 // getVmssVM gets virtualMachineScaleSetVM by nodeName from cache.
 // It returns cloudprovider.InstanceNotFound if node does not belong to any scale sets.
-func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm compute.VirtualMachineScaleSetVM, err error) {
+func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm computepreview.VirtualMachineScaleSetVM, err error) {
 	// Known node not managed by scale sets.
 	managedByAS, err := ss.isNodeManagedByAvailabilitySet(nodeName)
 	if err != nil {
@@ -125,12 +125,12 @@ func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm co
 		return ssName, instanceID, vm, cloudprovider.InstanceNotFound
 	}
 
-	return ssName, instanceID, *(cachedVM.(*compute.VirtualMachineScaleSetVM)), nil
+	return ssName, instanceID, *(cachedVM.(*computepreview.VirtualMachineScaleSetVM)), nil
 }
 
 // getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache.
 // The node must belong to one of scale sets.
-func (ss *scaleSet) getVmssVMByInstanceID(scaleSetName, instanceID string) (vm compute.VirtualMachineScaleSetVM, err error) {
+func (ss *scaleSet) getVmssVMByInstanceID(scaleSetName, instanceID string) (vm computepreview.VirtualMachineScaleSetVM, err error) {
 	vmName := ss.makeVmssVMName(scaleSetName, instanceID)
 	cachedVM, err := ss.vmssVMCache.Get(vmName)
 	if err != nil {
@@ -142,7 +142,7 @@ func (ss *scaleSet) getVmssVMByInstanceID(scaleSetName, instanceID string) (vm c
 		return vm, cloudprovider.InstanceNotFound
 	}
 
-	return *(cachedVM.(*compute.VirtualMachineScaleSetVM)), nil
+	return *(cachedVM.(*computepreview.VirtualMachineScaleSetVM)), nil
 }
 
 // GetInstanceIDByNodeName gets the cloud provider ID by node name.
@@ -259,7 +259,7 @@ func (ss *scaleSet) GetIPByNodeName(nodeName, vmSetName string) (string, error)
 }
 
 // This returns the full identifier of the primary NIC for the given VM.
-func (ss *scaleSet) getPrimaryInterfaceID(machine compute.VirtualMachineScaleSetVM) (string, error) {
+func (ss *scaleSet) getPrimaryInterfaceID(machine computepreview.VirtualMachineScaleSetVM) (string, error) {
 	if len(*machine.NetworkProfile.NetworkInterfaces) == 1 {
 		return *(*machine.NetworkProfile.NetworkInterfaces)[0].ID, nil
 	}
@@ -300,95 +300,36 @@ func extractScaleSetNameByExternalID(externalID string) (string, error) {
 	return matches[1], nil
 }
 
-// listScaleSetsWithRetry lists scale sets with exponential backoff retry.
+// listScaleSets lists all scale sets.
-func (ss *scaleSet) listScaleSetsWithRetry() ([]string, error) {
+func (ss *scaleSet) listScaleSets() ([]string, error) {
 	var err error
-	var result compute.VirtualMachineScaleSetListResult
+	ctx, cancel := getContextWithCancel()
-	allScaleSets := make([]string, 0)
+	defer cancel()
 
-	backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
+	allScaleSets, err := ss.VirtualMachineScaleSetsClient.List(ctx, ss.ResourceGroup)
-		result, err = ss.VirtualMachineScaleSetsClient.List(ss.ResourceGroup)
+	if err != nil {
-		if err != nil {
+		glog.Errorf("VirtualMachineScaleSetsClient.List failed: %v", err)
-			glog.Errorf("VirtualMachineScaleSetsClient.List for %v failed: %v", ss.ResourceGroup, err)
+		return nil, err
-			return false, err
-		}
-
-		return true, nil
-	})
-	if backoffError != nil {
-		return nil, backoffError
 	}
 
-	appendResults := (result.Value != nil && len(*result.Value) > 0)
+	ssNames := make([]string, len(allScaleSets))
-	for appendResults {
+	for i := range allScaleSets {
-		for _, scaleSet := range *result.Value {
+		ssNames[i] = *(allScaleSets[i].Name)
-			allScaleSets = append(allScaleSets, *scaleSet.Name)
-		}
-		appendResults = false
-
-		if result.NextLink != nil {
-			backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-				result, err = ss.VirtualMachineScaleSetsClient.ListNextResults(ss.ResourceGroup, result)
-				if err != nil {
-					glog.Errorf("VirtualMachineScaleSetsClient.ListNextResults for %v failed: %v", ss.ResourceGroup, err)
-					return false, err
-				}
-
-				return true, nil
-			})
-			if backoffError != nil {
-				return nil, backoffError
-			}
-
-			appendResults = (result.Value != nil && len(*result.Value) > 0)
-		}
-
 	}
 
-	return allScaleSets, nil
+	return ssNames, nil
 }
 
-// listScaleSetVMsWithRetry lists VMs belonging to the specified scale set with exponential backoff retry.
+// listScaleSetVMs lists VMs belonging to the specified scale set.
-func (ss *scaleSet) listScaleSetVMsWithRetry(scaleSetName string) ([]compute.VirtualMachineScaleSetVM, error) {
+func (ss *scaleSet) listScaleSetVMs(scaleSetName string) ([]computepreview.VirtualMachineScaleSetVM, error) {
 	var err error
-	var result compute.VirtualMachineScaleSetVMListResult
+	ctx, cancel := getContextWithCancel()
-	allVMs := make([]compute.VirtualMachineScaleSetVM, 0)
+	defer cancel()
-
-	backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-		result, err = ss.VirtualMachineScaleSetVMsClient.List(ss.ResourceGroup, scaleSetName, "", "", string(compute.InstanceView))
-		if err != nil {
-			glog.Errorf("VirtualMachineScaleSetVMsClient.List for %v failed: %v", scaleSetName, err)
-			return false, err
-		}
-
-		return true, nil
-	})
-	if backoffError != nil {
-		return nil, backoffError
-	}
-
-	appendResults := (result.Value != nil && len(*result.Value) > 0)
-	for appendResults {
-		allVMs = append(allVMs, *result.Value...)
-		appendResults = false
-
-		if result.NextLink != nil {
-			backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-				result, err = ss.VirtualMachineScaleSetVMsClient.ListNextResults(ss.ResourceGroup, result)
-				if err != nil {
-					glog.Errorf("VirtualMachineScaleSetVMsClient.ListNextResults for %v failed: %v", scaleSetName, err)
-					return false, err
-				}
-
-				return true, nil
-			})
-			if backoffError != nil {
-				return nil, backoffError
-			}
-
-			appendResults = (result.Value != nil && len(*result.Value) > 0)
-		}
 
+	allVMs, err := ss.VirtualMachineScaleSetVMsClient.List(ctx, ss.ResourceGroup, scaleSetName, "", "", string(computepreview.InstanceView))
+	if err != nil {
+		glog.Errorf("VirtualMachineScaleSetVMsClient.List failed: %v", err)
+		return nil, err
 	}
 
 	return allVMs, nil
@@ -517,8 +458,8 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName, vmSetName string) (network.Int
 }
 
 // getScaleSetWithRetry gets scale set with exponential backoff retry
-func (ss *scaleSet) getScaleSetWithRetry(name string) (compute.VirtualMachineScaleSet, bool, error) {
+func (ss *scaleSet) getScaleSetWithRetry(name string) (computepreview.VirtualMachineScaleSet, bool, error) {
-	var result compute.VirtualMachineScaleSet
+	var result computepreview.VirtualMachineScaleSet
 	var exists bool
 
 	err := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
@@ -531,7 +472,7 @@ func (ss *scaleSet) getScaleSetWithRetry(name string) (compute.VirtualMachineSca
 
 		if cached != nil {
 			exists = true
-			result = *(cached.(*compute.VirtualMachineScaleSet))
+			result = *(cached.(*computepreview.VirtualMachineScaleSet))
 		}
 
 		return true, nil
@@ -541,7 +482,7 @@ func (ss *scaleSet) getScaleSetWithRetry(name string) (compute.VirtualMachineSca
 }
 
 // getPrimaryNetworkConfiguration gets primary network interface configuration for scale sets.
-func (ss *scaleSet) getPrimaryNetworkConfiguration(networkConfigurationList *[]compute.VirtualMachineScaleSetNetworkConfiguration, scaleSetName string) (*compute.VirtualMachineScaleSetNetworkConfiguration, error) {
+func (ss *scaleSet) getPrimaryNetworkConfiguration(networkConfigurationList *[]computepreview.VirtualMachineScaleSetNetworkConfiguration, scaleSetName string) (*computepreview.VirtualMachineScaleSetNetworkConfiguration, error) {
 	networkConfigurations := *networkConfigurationList
 	if len(networkConfigurations) == 1 {
 		return &networkConfigurations[0], nil
@@ -557,7 +498,7 @@ func (ss *scaleSet) getPrimaryNetworkConfiguration(networkConfigurationList *[]c
 	return nil, fmt.Errorf("failed to find a primary network configuration for the scale set %q", scaleSetName)
 }
 
-func (ss *scaleSet) getPrimaryIPConfigForScaleSet(config *compute.VirtualMachineScaleSetNetworkConfiguration, scaleSetName string) (*compute.VirtualMachineScaleSetIPConfiguration, error) {
+func (ss *scaleSet) getPrimaryIPConfigForScaleSet(config *computepreview.VirtualMachineScaleSetNetworkConfiguration, scaleSetName string) (*computepreview.VirtualMachineScaleSetIPConfiguration, error) {
 	ipConfigurations := *config.IPConfigurations
 	if len(ipConfigurations) == 1 {
 		return &ipConfigurations[0], nil
@@ -574,24 +515,24 @@ func (ss *scaleSet) getPrimaryIPConfigForScaleSet(config *compute.VirtualMachine
 }
 
 // createOrUpdateVMSSWithRetry invokes ss.VirtualMachineScaleSetsClient.CreateOrUpdate with exponential backoff retry.
-func (ss *scaleSet) createOrUpdateVMSSWithRetry(virtualMachineScaleSet compute.VirtualMachineScaleSet) error {
+func (ss *scaleSet) createOrUpdateVMSSWithRetry(virtualMachineScaleSet computepreview.VirtualMachineScaleSet) error {
 	return wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-		respChan, errChan := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ss.ResourceGroup, *virtualMachineScaleSet.Name, virtualMachineScaleSet, nil)
+		ctx, cancel := getContextWithCancel()
-		resp := <-respChan
+		defer cancel()
-		err := <-errChan
+		resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, *virtualMachineScaleSet.Name, virtualMachineScaleSet)
 		glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%s): end", *virtualMachineScaleSet.Name)
-		return processRetryResponse(resp.Response, err)
+		return processHTTPRetryResponse(resp, err)
 	})
 }
 
 // updateVMSSInstancesWithRetry invokes ss.VirtualMachineScaleSetsClient.UpdateInstances with exponential backoff retry.
-func (ss *scaleSet) updateVMSSInstancesWithRetry(scaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) error {
+func (ss *scaleSet) updateVMSSInstancesWithRetry(scaleSetName string, vmInstanceIDs computepreview.VirtualMachineScaleSetVMInstanceRequiredIDs) error {
 	return wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-		respChan, errChan := ss.VirtualMachineScaleSetsClient.UpdateInstances(ss.ResourceGroup, scaleSetName, vmInstanceIDs, nil)
+		ctx, cancel := getContextWithCancel()
-		resp := <-respChan
+		defer cancel()
-		err := <-errChan
+		resp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(ctx, ss.ResourceGroup, scaleSetName, vmInstanceIDs)
 		glog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%s): end", scaleSetName)
-		return processRetryResponse(resp.Response, err)
+		return processHTTPRetryResponse(resp, err)
 	})
 }
 
@@ -624,7 +565,7 @@ func (ss *scaleSet) EnsureHostsInPool(serviceName string, nodes []*v1.Node, back
 
 	// Update primary IP configuration's LoadBalancerBackendAddressPools.
 	foundPool := false
-	newBackendPools := []compute.SubResource{}
+	newBackendPools := []computepreview.SubResource{}
 	if primaryIPConfiguration.LoadBalancerBackendAddressPools != nil {
 		newBackendPools = *primaryIPConfiguration.LoadBalancerBackendAddressPools
 	}
@@ -636,17 +577,17 @@ func (ss *scaleSet) EnsureHostsInPool(serviceName string, nodes []*v1.Node, back
 	}
 	if !foundPool {
 		newBackendPools = append(newBackendPools,
-			compute.SubResource{
+			computepreview.SubResource{
 				ID: to.StringPtr(backendPoolID),
 			})
 		primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools
 
+		ctx, cancel := getContextWithCancel()
+		defer cancel()
 		glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s): scale set (%s) - updating", serviceName, vmSetName)
-		respChan, errChan := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ss.ResourceGroup, vmSetName, virtualMachineScaleSet, nil)
+		resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, vmSetName, virtualMachineScaleSet)
-		resp := <-respChan
-		err := <-errChan
 		glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName)
-		if ss.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+		if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
 			glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s): scale set (%s) - updating, err=%v", serviceName, vmSetName, err)
 			retryErr := ss.createOrUpdateVMSSWithRetry(virtualMachineScaleSet)
 			if retryErr != nil {
@@ -682,14 +623,14 @@ func (ss *scaleSet) EnsureHostsInPool(serviceName string, nodes []*v1.Node, back
 	}
 
 	// Update instances to latest VMSS model.
-	vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{
+	vmInstanceIDs := computepreview.VirtualMachineScaleSetVMInstanceRequiredIDs{
 		InstanceIds: &instanceIDs,
 	}
-	respChan, errChan := ss.VirtualMachineScaleSetsClient.UpdateInstances(ss.ResourceGroup, vmSetName, vmInstanceIDs, nil)
+	ctx, cancel := getContextWithCancel()
-	resp := <-respChan
+	defer cancel()
-	err = <-errChan
+	instanceResp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(ctx, ss.ResourceGroup, vmSetName, vmInstanceIDs)
-	glog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%q): end", vmSetName)
+	glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName)
-	if ss.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+	if ss.CloudProviderBackoff && shouldRetryHTTPRequest(instanceResp, err) {
 		glog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances for service (%s): scale set (%s) - updating, err=%v", serviceName, vmSetName, err)
 		retryErr := ss.updateVMSSInstancesWithRetry(vmSetName, vmInstanceIDs)
 		if retryErr != nil {
@@ -734,7 +675,7 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(poolID, vmSetName string) error {
 		return nil
 	}
 	existingBackendPools := *primaryIPConfiguration.LoadBalancerBackendAddressPools
-	newBackendPools := []compute.SubResource{}
+	newBackendPools := []computepreview.SubResource{}
 	foundPool := false
 	for i := len(existingBackendPools) - 1; i >= 0; i-- {
 		curPool := existingBackendPools[i]
@@ -752,11 +693,11 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(poolID, vmSetName string) error {
 	// Update scale set with backoff.
 	primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools
 	glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating", vmSetName)
-	respChan, errChan := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ss.ResourceGroup, vmSetName, virtualMachineScaleSet, nil)
+	ctx, cancel := getContextWithCancel()
-	resp := <-respChan
+	defer cancel()
-	err = <-errChan
+	resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, vmSetName, virtualMachineScaleSet)
 	glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName)
-	if ss.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+	if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
 		glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating, err=%v", vmSetName, err)
 		retryErr := ss.createOrUpdateVMSSWithRetry(virtualMachineScaleSet)
 		if retryErr != nil {
@@ -770,14 +711,12 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(poolID, vmSetName string) error {
 
 	// Update instances to latest VMSS model.
 	instanceIDs := []string{"*"}
-	vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{
+	vmInstanceIDs := computepreview.VirtualMachineScaleSetVMInstanceRequiredIDs{
 		InstanceIds: &instanceIDs,
 	}
-	updateRespChan, errChan := ss.VirtualMachineScaleSetsClient.UpdateInstances(ss.ResourceGroup, vmSetName, vmInstanceIDs, nil)
+	instanceResp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(ctx, ss.ResourceGroup, vmSetName, vmInstanceIDs)
-	updateResp := <-updateRespChan
-	err = <-errChan
 	glog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%q): end", vmSetName)
-	if ss.CloudProviderBackoff && shouldRetryAPIRequest(updateResp.Response, err) {
+	if ss.CloudProviderBackoff && shouldRetryHTTPRequest(instanceResp, err) {
 		glog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances scale set (%s) - updating, err=%v", vmSetName, err)
 		retryErr := ss.updateVMSSInstancesWithRetry(vmSetName, vmInstanceIDs)
 		if retryErr != nil {
@@ -793,11 +732,9 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(poolID, vmSetName string) error {
 	// TODO: remove this workaround when figuring out the root cause.
 	if len(newBackendPools) == 0 {
 		glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating second time", vmSetName)
-		respChan, errChan = ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ss.ResourceGroup, vmSetName, virtualMachineScaleSet, nil)
+		resp, err = ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, vmSetName, virtualMachineScaleSet)
-		resp = <-respChan
-		err = <-errChan
 		glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName)
-		if ss.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+		if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
 			glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating, err=%v", vmSetName, err)
 			retryErr := ss.createOrUpdateVMSSWithRetry(virtualMachineScaleSet)
 			if retryErr != nil {

pkg/cloudprovider/providers/azure/azure_vmss_cache.go

@@ -58,7 +58,9 @@ func (ss *scaleSet) extractVmssVMName(name string) (string, string, error) {
 
 func (ss *scaleSet) newVmssCache() (*timedCache, error) {
 	getter := func(key string) (interface{}, error) {
-		result, err := ss.VirtualMachineScaleSetsClient.Get(ss.ResourceGroup, key)
+		ctx, cancel := getContextWithCancel()
+		defer cancel()
+		result, err := ss.VirtualMachineScaleSetsClient.Get(ctx, ss.ResourceGroup, key)
 		exists, realErr := checkResourceExistsFromError(err)
 		if realErr != nil {
 			return nil, realErr
@@ -76,14 +78,14 @@ func (ss *scaleSet) newVmssCache() (*timedCache, error) {
 
 func (ss *scaleSet) newNodeNameToScaleSetMappingCache() (*timedCache, error) {
 	getter := func(key string) (interface{}, error) {
-		scaleSetNames, err := ss.listScaleSetsWithRetry()
+		scaleSetNames, err := ss.listScaleSets()
 		if err != nil {
 			return nil, err
 		}
 
 		localCache := make(nodeNameToScaleSetMapping)
 		for _, ssName := range scaleSetNames {
-			vms, err := ss.listScaleSetVMsWithRetry(ssName)
+			vms, err := ss.listScaleSetVMs(ssName)
 			if err != nil {
 				return nil, err
 			}
@@ -136,7 +138,9 @@ func (ss *scaleSet) newVmssVMCache() (*timedCache, error) {
 			return nil, nil
 		}
 
-		result, err := ss.VirtualMachineScaleSetVMsClient.Get(ss.ResourceGroup, ssName, instanceID)
+		ctx, cancel := getContextWithCancel()
+		defer cancel()
+		result, err := ss.VirtualMachineScaleSetVMsClient.Get(ctx, ss.ResourceGroup, ssName, instanceID)
 		exists, realErr := checkResourceExistsFromError(err)
 		if realErr != nil {
 			return nil, realErr