diff --git a/pkg/cloudprovider/providers/azure/azure_backoff.go b/pkg/cloudprovider/providers/azure/azure_backoff.go index 39d2a291b5a..ab5707656ad 100644 --- a/pkg/cloudprovider/providers/azure/azure_backoff.go +++ b/pkg/cloudprovider/providers/azure/azure_backoff.go @@ -17,13 +17,17 @@ limitations under the License. package azure import ( - "k8s.io/apimachinery/pkg/util/wait" + "context" + "net/http" "github.com/Azure/azure-sdk-for-go/arm/compute" "github.com/Azure/azure-sdk-for-go/arm/network" + computepreview "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute" "github.com/Azure/go-autorest/autorest" "github.com/golang/glog" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" ) // requestBackoff if backoff is disabled in cloud provider it @@ -345,6 +349,15 @@ func (az *Cloud) CreateOrUpdateVMWithRetry(vmName string, newVM compute.VirtualM }) } +// UpdateVmssVMWithRetry invokes az.VirtualMachineScaleSetVMsClient.Update with exponential backoff retry +func (az *Cloud) UpdateVmssVMWithRetry(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, parameters computepreview.VirtualMachineScaleSetVM) error { + return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { + resp, err := az.VirtualMachineScaleSetVMsClient.Update(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters) + glog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s,%s): end", VMScaleSetName, instanceID) + return processHTTPRetryResponse(resp, err) + }) +} + // A wait.ConditionFunc function to deal with common HTTP backoff response conditions func processRetryResponse(resp autorest.Response, err error) (bool, error) { if isSuccessHTTPResponse(resp) { @@ -380,3 +393,36 @@ func isSuccessHTTPResponse(resp autorest.Response) bool { } return false } + +func shouldRetryHTTPRequest(resp *http.Response, err error) bool { + if err != nil { + return true + } + + if resp != nil { + // HTTP 4xx or 5xx suggests we should retry + if 399 < resp.StatusCode && resp.StatusCode < 600 { + return true + } + } + + return false +} + +func processHTTPRetryResponse(resp *http.Response, err error) (bool, error) { + if resp != nil { + // HTTP 2xx suggests a successful response + if 199 < resp.StatusCode && resp.StatusCode < 300 { + return true, nil + } + } + + if shouldRetryHTTPRequest(resp, err) { + glog.Errorf("backoff: failure, will retry, HTTP response=%d, err=%v", resp.StatusCode, err) + // suppress the error object so that backoff process continues + return false, nil + } + + // Fall-through: stop periodic backoff + return true, nil +} diff --git a/pkg/cloudprovider/providers/azure/azure_client.go b/pkg/cloudprovider/providers/azure/azure_client.go index 1887162b946..1516339724c 100644 --- a/pkg/cloudprovider/providers/azure/azure_client.go +++ b/pkg/cloudprovider/providers/azure/azure_client.go @@ -140,6 +140,10 @@ type azVirtualMachinesClient struct { rateLimiter flowcontrol.RateLimiter } +func getContextWithCancel() (context.Context, context.CancelFunc) { + return context.WithCancel(context.Background()) +} + func newAzVirtualMachinesClient(config *azClientConfig) *azVirtualMachinesClient { virtualMachinesClient := compute.NewVirtualMachinesClient(config.subscriptionID) virtualMachinesClient.BaseURI = config.resourceManagerEndpoint diff --git a/pkg/cloudprovider/providers/azure/azure_vmss.go b/pkg/cloudprovider/providers/azure/azure_vmss.go index c16189f608c..31883791350 100644 --- a/pkg/cloudprovider/providers/azure/azure_vmss.go +++ b/pkg/cloudprovider/providers/azure/azure_vmss.go @@ -24,8 +24,8 @@ import ( "strconv" "strings" - "github.com/Azure/azure-sdk-for-go/arm/compute" "github.com/Azure/azure-sdk-for-go/arm/network" + computepreview "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/glog" @@ -89,7 +89,7 @@ func newScaleSet(az *Cloud) (VMSet, error) { // getVmssVM gets virtualMachineScaleSetVM by nodeName from cache. // It returns cloudprovider.InstanceNotFound if node does not belong to any scale sets. -func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm compute.VirtualMachineScaleSetVM, err error) { +func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm computepreview.VirtualMachineScaleSetVM, err error) { // Known node not managed by scale sets. managedByAS, err := ss.isNodeManagedByAvailabilitySet(nodeName) if err != nil { @@ -125,12 +125,12 @@ func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm co return ssName, instanceID, vm, cloudprovider.InstanceNotFound } - return ssName, instanceID, *(cachedVM.(*compute.VirtualMachineScaleSetVM)), nil + return ssName, instanceID, *(cachedVM.(*computepreview.VirtualMachineScaleSetVM)), nil } // getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache. // The node must belong to one of scale sets. -func (ss *scaleSet) getVmssVMByInstanceID(scaleSetName, instanceID string) (vm compute.VirtualMachineScaleSetVM, err error) { +func (ss *scaleSet) getVmssVMByInstanceID(scaleSetName, instanceID string) (vm computepreview.VirtualMachineScaleSetVM, err error) { vmName := ss.makeVmssVMName(scaleSetName, instanceID) cachedVM, err := ss.vmssVMCache.Get(vmName) if err != nil { @@ -142,7 +142,7 @@ func (ss *scaleSet) getVmssVMByInstanceID(scaleSetName, instanceID string) (vm c return vm, cloudprovider.InstanceNotFound } - return *(cachedVM.(*compute.VirtualMachineScaleSetVM)), nil + return *(cachedVM.(*computepreview.VirtualMachineScaleSetVM)), nil } // GetInstanceIDByNodeName gets the cloud provider ID by node name. @@ -259,7 +259,7 @@ func (ss *scaleSet) GetIPByNodeName(nodeName, vmSetName string) (string, error) } // This returns the full identifier of the primary NIC for the given VM. -func (ss *scaleSet) getPrimaryInterfaceID(machine compute.VirtualMachineScaleSetVM) (string, error) { +func (ss *scaleSet) getPrimaryInterfaceID(machine computepreview.VirtualMachineScaleSetVM) (string, error) { if len(*machine.NetworkProfile.NetworkInterfaces) == 1 { return *(*machine.NetworkProfile.NetworkInterfaces)[0].ID, nil } @@ -300,95 +300,36 @@ func extractScaleSetNameByExternalID(externalID string) (string, error) { return matches[1], nil } -// listScaleSetsWithRetry lists scale sets with exponential backoff retry. -func (ss *scaleSet) listScaleSetsWithRetry() ([]string, error) { +// listScaleSets lists all scale sets. +func (ss *scaleSet) listScaleSets() ([]string, error) { var err error - var result compute.VirtualMachineScaleSetListResult - allScaleSets := make([]string, 0) + ctx, cancel := getContextWithCancel() + defer cancel() - backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) { - result, err = ss.VirtualMachineScaleSetsClient.List(ss.ResourceGroup) - if err != nil { - glog.Errorf("VirtualMachineScaleSetsClient.List for %v failed: %v", ss.ResourceGroup, err) - return false, err - } - - return true, nil - }) - if backoffError != nil { - return nil, backoffError + allScaleSets, err := ss.VirtualMachineScaleSetsClient.List(ctx, ss.ResourceGroup) + if err != nil { + glog.Errorf("VirtualMachineScaleSetsClient.List failed: %v", err) + return nil, err } - appendResults := (result.Value != nil && len(*result.Value) > 0) - for appendResults { - for _, scaleSet := range *result.Value { - allScaleSets = append(allScaleSets, *scaleSet.Name) - } - appendResults = false - - if result.NextLink != nil { - backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) { - result, err = ss.VirtualMachineScaleSetsClient.ListNextResults(ss.ResourceGroup, result) - if err != nil { - glog.Errorf("VirtualMachineScaleSetsClient.ListNextResults for %v failed: %v", ss.ResourceGroup, err) - return false, err - } - - return true, nil - }) - if backoffError != nil { - return nil, backoffError - } - - appendResults = (result.Value != nil && len(*result.Value) > 0) - } - + ssNames := make([]string, len(allScaleSets)) + for i := range allScaleSets { + ssNames[i] = *(allScaleSets[i].Name) } - return allScaleSets, nil + return ssNames, nil } -// listScaleSetVMsWithRetry lists VMs belonging to the specified scale set with exponential backoff retry. -func (ss *scaleSet) listScaleSetVMsWithRetry(scaleSetName string) ([]compute.VirtualMachineScaleSetVM, error) { +// listScaleSetVMs lists VMs belonging to the specified scale set. +func (ss *scaleSet) listScaleSetVMs(scaleSetName string) ([]computepreview.VirtualMachineScaleSetVM, error) { var err error - var result compute.VirtualMachineScaleSetVMListResult - allVMs := make([]compute.VirtualMachineScaleSetVM, 0) - - backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) { - result, err = ss.VirtualMachineScaleSetVMsClient.List(ss.ResourceGroup, scaleSetName, "", "", string(compute.InstanceView)) - if err != nil { - glog.Errorf("VirtualMachineScaleSetVMsClient.List for %v failed: %v", scaleSetName, err) - return false, err - } - - return true, nil - }) - if backoffError != nil { - return nil, backoffError - } - - appendResults := (result.Value != nil && len(*result.Value) > 0) - for appendResults { - allVMs = append(allVMs, *result.Value...) - appendResults = false - - if result.NextLink != nil { - backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) { - result, err = ss.VirtualMachineScaleSetVMsClient.ListNextResults(ss.ResourceGroup, result) - if err != nil { - glog.Errorf("VirtualMachineScaleSetVMsClient.ListNextResults for %v failed: %v", scaleSetName, err) - return false, err - } - - return true, nil - }) - if backoffError != nil { - return nil, backoffError - } - - appendResults = (result.Value != nil && len(*result.Value) > 0) - } + ctx, cancel := getContextWithCancel() + defer cancel() + allVMs, err := ss.VirtualMachineScaleSetVMsClient.List(ctx, ss.ResourceGroup, scaleSetName, "", "", string(computepreview.InstanceView)) + if err != nil { + glog.Errorf("VirtualMachineScaleSetVMsClient.List failed: %v", err) + return nil, err } return allVMs, nil @@ -517,8 +458,8 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName, vmSetName string) (network.Int } // getScaleSetWithRetry gets scale set with exponential backoff retry -func (ss *scaleSet) getScaleSetWithRetry(name string) (compute.VirtualMachineScaleSet, bool, error) { - var result compute.VirtualMachineScaleSet +func (ss *scaleSet) getScaleSetWithRetry(name string) (computepreview.VirtualMachineScaleSet, bool, error) { + var result computepreview.VirtualMachineScaleSet var exists bool err := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) { @@ -531,7 +472,7 @@ func (ss *scaleSet) getScaleSetWithRetry(name string) (compute.VirtualMachineSca if cached != nil { exists = true - result = *(cached.(*compute.VirtualMachineScaleSet)) + result = *(cached.(*computepreview.VirtualMachineScaleSet)) } return true, nil @@ -541,7 +482,7 @@ func (ss *scaleSet) getScaleSetWithRetry(name string) (compute.VirtualMachineSca } // getPrimaryNetworkConfiguration gets primary network interface configuration for scale sets. -func (ss *scaleSet) getPrimaryNetworkConfiguration(networkConfigurationList *[]compute.VirtualMachineScaleSetNetworkConfiguration, scaleSetName string) (*compute.VirtualMachineScaleSetNetworkConfiguration, error) { +func (ss *scaleSet) getPrimaryNetworkConfiguration(networkConfigurationList *[]computepreview.VirtualMachineScaleSetNetworkConfiguration, scaleSetName string) (*computepreview.VirtualMachineScaleSetNetworkConfiguration, error) { networkConfigurations := *networkConfigurationList if len(networkConfigurations) == 1 { return &networkConfigurations[0], nil @@ -557,7 +498,7 @@ func (ss *scaleSet) getPrimaryNetworkConfiguration(networkConfigurationList *[]c return nil, fmt.Errorf("failed to find a primary network configuration for the scale set %q", scaleSetName) } -func (ss *scaleSet) getPrimaryIPConfigForScaleSet(config *compute.VirtualMachineScaleSetNetworkConfiguration, scaleSetName string) (*compute.VirtualMachineScaleSetIPConfiguration, error) { +func (ss *scaleSet) getPrimaryIPConfigForScaleSet(config *computepreview.VirtualMachineScaleSetNetworkConfiguration, scaleSetName string) (*computepreview.VirtualMachineScaleSetIPConfiguration, error) { ipConfigurations := *config.IPConfigurations if len(ipConfigurations) == 1 { return &ipConfigurations[0], nil @@ -574,24 +515,24 @@ func (ss *scaleSet) getPrimaryIPConfigForScaleSet(config *compute.VirtualMachine } // createOrUpdateVMSSWithRetry invokes ss.VirtualMachineScaleSetsClient.CreateOrUpdate with exponential backoff retry. -func (ss *scaleSet) createOrUpdateVMSSWithRetry(virtualMachineScaleSet compute.VirtualMachineScaleSet) error { +func (ss *scaleSet) createOrUpdateVMSSWithRetry(virtualMachineScaleSet computepreview.VirtualMachineScaleSet) error { return wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) { - respChan, errChan := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ss.ResourceGroup, *virtualMachineScaleSet.Name, virtualMachineScaleSet, nil) - resp := <-respChan - err := <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, *virtualMachineScaleSet.Name, virtualMachineScaleSet) glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%s): end", *virtualMachineScaleSet.Name) - return processRetryResponse(resp.Response, err) + return processHTTPRetryResponse(resp, err) }) } // updateVMSSInstancesWithRetry invokes ss.VirtualMachineScaleSetsClient.UpdateInstances with exponential backoff retry. -func (ss *scaleSet) updateVMSSInstancesWithRetry(scaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) error { +func (ss *scaleSet) updateVMSSInstancesWithRetry(scaleSetName string, vmInstanceIDs computepreview.VirtualMachineScaleSetVMInstanceRequiredIDs) error { return wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) { - respChan, errChan := ss.VirtualMachineScaleSetsClient.UpdateInstances(ss.ResourceGroup, scaleSetName, vmInstanceIDs, nil) - resp := <-respChan - err := <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + resp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(ctx, ss.ResourceGroup, scaleSetName, vmInstanceIDs) glog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%s): end", scaleSetName) - return processRetryResponse(resp.Response, err) + return processHTTPRetryResponse(resp, err) }) } @@ -624,7 +565,7 @@ func (ss *scaleSet) EnsureHostsInPool(serviceName string, nodes []*v1.Node, back // Update primary IP configuration's LoadBalancerBackendAddressPools. foundPool := false - newBackendPools := []compute.SubResource{} + newBackendPools := []computepreview.SubResource{} if primaryIPConfiguration.LoadBalancerBackendAddressPools != nil { newBackendPools = *primaryIPConfiguration.LoadBalancerBackendAddressPools } @@ -636,17 +577,17 @@ func (ss *scaleSet) EnsureHostsInPool(serviceName string, nodes []*v1.Node, back } if !foundPool { newBackendPools = append(newBackendPools, - compute.SubResource{ + computepreview.SubResource{ ID: to.StringPtr(backendPoolID), }) primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools + ctx, cancel := getContextWithCancel() + defer cancel() glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s): scale set (%s) - updating", serviceName, vmSetName) - respChan, errChan := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ss.ResourceGroup, vmSetName, virtualMachineScaleSet, nil) - resp := <-respChan - err := <-errChan + resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, vmSetName, virtualMachineScaleSet) glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName) - if ss.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) { + if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate for service (%s): scale set (%s) - updating, err=%v", serviceName, vmSetName, err) retryErr := ss.createOrUpdateVMSSWithRetry(virtualMachineScaleSet) if retryErr != nil { @@ -682,14 +623,14 @@ func (ss *scaleSet) EnsureHostsInPool(serviceName string, nodes []*v1.Node, back } // Update instances to latest VMSS model. - vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ + vmInstanceIDs := computepreview.VirtualMachineScaleSetVMInstanceRequiredIDs{ InstanceIds: &instanceIDs, } - respChan, errChan := ss.VirtualMachineScaleSetsClient.UpdateInstances(ss.ResourceGroup, vmSetName, vmInstanceIDs, nil) - resp := <-respChan - err = <-errChan - glog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%q): end", vmSetName) - if ss.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) { + ctx, cancel := getContextWithCancel() + defer cancel() + instanceResp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(ctx, ss.ResourceGroup, vmSetName, vmInstanceIDs) + glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName) + if ss.CloudProviderBackoff && shouldRetryHTTPRequest(instanceResp, err) { glog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances for service (%s): scale set (%s) - updating, err=%v", serviceName, vmSetName, err) retryErr := ss.updateVMSSInstancesWithRetry(vmSetName, vmInstanceIDs) if retryErr != nil { @@ -734,7 +675,7 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(poolID, vmSetName string) error { return nil } existingBackendPools := *primaryIPConfiguration.LoadBalancerBackendAddressPools - newBackendPools := []compute.SubResource{} + newBackendPools := []computepreview.SubResource{} foundPool := false for i := len(existingBackendPools) - 1; i >= 0; i-- { curPool := existingBackendPools[i] @@ -752,11 +693,11 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(poolID, vmSetName string) error { // Update scale set with backoff. primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating", vmSetName) - respChan, errChan := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ss.ResourceGroup, vmSetName, virtualMachineScaleSet, nil) - resp := <-respChan - err = <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + resp, err := ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, vmSetName, virtualMachineScaleSet) glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName) - if ss.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) { + if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating, err=%v", vmSetName, err) retryErr := ss.createOrUpdateVMSSWithRetry(virtualMachineScaleSet) if retryErr != nil { @@ -770,14 +711,12 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(poolID, vmSetName string) error { // Update instances to latest VMSS model. instanceIDs := []string{"*"} - vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ + vmInstanceIDs := computepreview.VirtualMachineScaleSetVMInstanceRequiredIDs{ InstanceIds: &instanceIDs, } - updateRespChan, errChan := ss.VirtualMachineScaleSetsClient.UpdateInstances(ss.ResourceGroup, vmSetName, vmInstanceIDs, nil) - updateResp := <-updateRespChan - err = <-errChan + instanceResp, err := ss.VirtualMachineScaleSetsClient.UpdateInstances(ctx, ss.ResourceGroup, vmSetName, vmInstanceIDs) glog.V(10).Infof("VirtualMachineScaleSetsClient.UpdateInstances(%q): end", vmSetName) - if ss.CloudProviderBackoff && shouldRetryAPIRequest(updateResp.Response, err) { + if ss.CloudProviderBackoff && shouldRetryHTTPRequest(instanceResp, err) { glog.V(2).Infof("VirtualMachineScaleSetsClient.UpdateInstances scale set (%s) - updating, err=%v", vmSetName, err) retryErr := ss.updateVMSSInstancesWithRetry(vmSetName, vmInstanceIDs) if retryErr != nil { @@ -793,11 +732,9 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(poolID, vmSetName string) error { // TODO: remove this workaround when figuring out the root cause. if len(newBackendPools) == 0 { glog.V(3).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating second time", vmSetName) - respChan, errChan = ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ss.ResourceGroup, vmSetName, virtualMachineScaleSet, nil) - resp = <-respChan - err = <-errChan + resp, err = ss.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, ss.ResourceGroup, vmSetName, virtualMachineScaleSet) glog.V(10).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate(%q): end", vmSetName) - if ss.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) { + if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { glog.V(2).Infof("VirtualMachineScaleSetsClient.CreateOrUpdate: scale set (%s) - updating, err=%v", vmSetName, err) retryErr := ss.createOrUpdateVMSSWithRetry(virtualMachineScaleSet) if retryErr != nil { diff --git a/pkg/cloudprovider/providers/azure/azure_vmss_cache.go b/pkg/cloudprovider/providers/azure/azure_vmss_cache.go index c6683f460db..8aad8425e61 100644 --- a/pkg/cloudprovider/providers/azure/azure_vmss_cache.go +++ b/pkg/cloudprovider/providers/azure/azure_vmss_cache.go @@ -58,7 +58,9 @@ func (ss *scaleSet) extractVmssVMName(name string) (string, string, error) { func (ss *scaleSet) newVmssCache() (*timedCache, error) { getter := func(key string) (interface{}, error) { - result, err := ss.VirtualMachineScaleSetsClient.Get(ss.ResourceGroup, key) + ctx, cancel := getContextWithCancel() + defer cancel() + result, err := ss.VirtualMachineScaleSetsClient.Get(ctx, ss.ResourceGroup, key) exists, realErr := checkResourceExistsFromError(err) if realErr != nil { return nil, realErr @@ -76,14 +78,14 @@ func (ss *scaleSet) newVmssCache() (*timedCache, error) { func (ss *scaleSet) newNodeNameToScaleSetMappingCache() (*timedCache, error) { getter := func(key string) (interface{}, error) { - scaleSetNames, err := ss.listScaleSetsWithRetry() + scaleSetNames, err := ss.listScaleSets() if err != nil { return nil, err } localCache := make(nodeNameToScaleSetMapping) for _, ssName := range scaleSetNames { - vms, err := ss.listScaleSetVMsWithRetry(ssName) + vms, err := ss.listScaleSetVMs(ssName) if err != nil { return nil, err } @@ -136,7 +138,9 @@ func (ss *scaleSet) newVmssVMCache() (*timedCache, error) { return nil, nil } - result, err := ss.VirtualMachineScaleSetVMsClient.Get(ss.ResourceGroup, ssName, instanceID) + ctx, cancel := getContextWithCancel() + defer cancel() + result, err := ss.VirtualMachineScaleSetVMsClient.Get(ctx, ss.ResourceGroup, ssName, instanceID) exists, realErr := checkResourceExistsFromError(err) if realErr != nil { return nil, realErr