github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 23:15:14 +00:00
Code Issues Projects Releases Wiki Activity

add retry for detach azure disk

Browse Source 
add more logging info in detach disk
This commit is contained in:
andyzhangx 2019-02-22 05:13:16 +00:00
parent 4f4695fefe
commit 8c53db0941
3 changed files with 38 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/cloudprovider/providers/azure/azure_backoff.go
View File

@ -511,6 +511,18 @@ func (az *Cloud) CreateOrUpdateVMWithRetry(resourceGroup, vmName string, newVM c
}) })
} }
// UpdateVmssVMWithRetry invokes az.VirtualMachineScaleSetVMsClient.Update with exponential backoff retry
func (az *Cloud) UpdateVmssVMWithRetry(resourceGroupName string, VMScaleSetName string, instanceID string, parameters compute.VirtualMachineScaleSetVM) error {
return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
ctx, cancel := getContextWithCancel()
defer cancel()
resp, err := az.VirtualMachineScaleSetVMsClient.Update(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters)
klog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s,%s): end", VMScaleSetName, instanceID)
return az.processHTTPRetryResponse(nil, "", resp, err)
})
}
// isSuccessHTTPResponse determines if the response from an HTTP request suggests success // isSuccessHTTPResponse determines if the response from an HTTP request suggests success
func isSuccessHTTPResponse(resp *http.Response) bool { func isSuccessHTTPResponse(resp *http.Response) bool {
if resp == nil { if resp == nil {

19
pkg/cloudprovider/providers/azure/azure_controller_standard.go
View File

@ -146,12 +146,21 @@ func (as *availabilitySet) DetachDiskByName(diskName, diskURI string, nodeName t
// Invalidate the cache right after updating // Invalidate the cache right after updating
defer as.cloud.vmCache.Delete(vmName) defer as.cloud.vmCache.Delete(vmName)
_, err = as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM) resp, err := as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM)
if err != nil { if as.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
klog.Errorf("azureDisk - detach disk(%s) failed, err: %v", diskName, err) klog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, vmName, diskName, diskURI, err)
} else { retryErr := as.CreateOrUpdateVMWithRetry(nodeResourceGroup, vmName, newVM)
klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) if retryErr != nil {
err = retryErr
klog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, vmName, diskName, diskURI, err)
}
} }
if err != nil {
klog.Errorf("azureDisk - detach disk(%s, %s)) failed, err: %v", diskName, diskURI, err)
} else {
klog.V(2).Infof("azureDisk - detach disk(%s, %s) succeeded", diskName, diskURI)
}
return err return err
} }

16
pkg/cloudprovider/providers/azure/azure_controller_vmss.go
View File

@ -155,12 +155,20 @@ func (ss *scaleSet) DetachDiskByName(diskName, diskURI string, nodeName types.No
key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID))
defer ss.vmssVMCache.Delete(key) defer ss.vmssVMCache.Delete(key)
klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, nodeName, diskName) klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s, %s)", nodeResourceGroup, nodeName, diskName, diskURI)
_, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM) resp, err := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM)
if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
klog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, nodeName, diskName, diskURI, err)
retryErr := ss.UpdateVmssVMWithRetry(nodeResourceGroup, ssName, instanceID, newVM)
if retryErr != nil {
err = retryErr
klog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, nodeName, diskName, diskURI, err)
}
}
if err != nil { if err != nil {
klog.Errorf("azureDisk - detach disk(%s) from %s failed, err: %v", diskName, nodeName, err) klog.Errorf("azureDisk - detach disk(%s, %s) from %s failed, err: %v", diskName, diskURI, nodeName, err)
} else { } else {
klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) klog.V(2).Infof("azureDisk - detach disk(%s, %s) succeeded", diskName, diskURI)
} }
return err return err