mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
fix race condition when attach/delete disk
fix comments
This commit is contained in:
parent
b26467b344
commit
b6afc8604c
@ -23,6 +23,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"path"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute"
|
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute"
|
||||||
@ -65,7 +66,9 @@ type controllerCommon struct {
|
|||||||
location string
|
location string
|
||||||
storageEndpointSuffix string
|
storageEndpointSuffix string
|
||||||
resourceGroup string
|
resourceGroup string
|
||||||
cloud *Cloud
|
// store disk URI when disk is in attaching or detaching process
|
||||||
|
diskAttachDetachMap sync.Map
|
||||||
|
cloud *Cloud
|
||||||
}
|
}
|
||||||
|
|
||||||
// getNodeVMSet gets the VMSet interface based on config.VMType and the real virtual machine type.
|
// getNodeVMSet gets the VMSet interface based on config.VMType and the real virtual machine type.
|
||||||
@ -151,6 +154,8 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
|
|||||||
}
|
}
|
||||||
|
|
||||||
klog.V(2).Infof("Trying to attach volume %q lun %d to node %q.", diskURI, lun, nodeName)
|
klog.V(2).Infof("Trying to attach volume %q lun %d to node %q.", diskURI, lun, nodeName)
|
||||||
|
c.diskAttachDetachMap.Store(strings.ToLower(diskURI), "attaching")
|
||||||
|
defer c.diskAttachDetachMap.Delete(strings.ToLower(diskURI))
|
||||||
return lun, vmset.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode, diskEncryptionSetID)
|
return lun, vmset.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode, diskEncryptionSetID)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -177,14 +182,18 @@ func (c *controllerCommon) DetachDisk(diskName, diskURI string, nodeName types.N
|
|||||||
|
|
||||||
// make the lock here as small as possible
|
// make the lock here as small as possible
|
||||||
diskOpMutex.LockKey(instanceid)
|
diskOpMutex.LockKey(instanceid)
|
||||||
|
c.diskAttachDetachMap.Store(strings.ToLower(diskURI), "detaching")
|
||||||
resp, err := vmset.DetachDisk(diskName, diskURI, nodeName)
|
resp, err := vmset.DetachDisk(diskName, diskURI, nodeName)
|
||||||
|
c.diskAttachDetachMap.Delete(strings.ToLower(diskURI))
|
||||||
diskOpMutex.UnlockKey(instanceid)
|
diskOpMutex.UnlockKey(instanceid)
|
||||||
|
|
||||||
if c.cloud.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
|
if c.cloud.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
|
||||||
klog.V(2).Infof("azureDisk - update backing off: detach disk(%s, %s), err: %v", diskName, diskURI, err)
|
klog.V(2).Infof("azureDisk - update backing off: detach disk(%s, %s), err: %v", diskName, diskURI, err)
|
||||||
retryErr := kwait.ExponentialBackoff(c.cloud.RequestBackoff(), func() (bool, error) {
|
retryErr := kwait.ExponentialBackoff(c.cloud.RequestBackoff(), func() (bool, error) {
|
||||||
diskOpMutex.LockKey(instanceid)
|
diskOpMutex.LockKey(instanceid)
|
||||||
|
c.diskAttachDetachMap.Store(strings.ToLower(diskURI), "detaching")
|
||||||
resp, err := vmset.DetachDisk(diskName, diskURI, nodeName)
|
resp, err := vmset.DetachDisk(diskName, diskURI, nodeName)
|
||||||
|
c.diskAttachDetachMap.Delete(strings.ToLower(diskURI))
|
||||||
diskOpMutex.UnlockKey(instanceid)
|
diskOpMutex.UnlockKey(instanceid)
|
||||||
return c.cloud.processHTTPRetryResponse(nil, "", resp, err)
|
return c.cloud.processHTTPRetryResponse(nil, "", resp, err)
|
||||||
})
|
})
|
||||||
@ -226,9 +235,13 @@ func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.N
|
|||||||
if disk.Lun != nil && (disk.Name != nil && diskName != "" && strings.EqualFold(*disk.Name, diskName)) ||
|
if disk.Lun != nil && (disk.Name != nil && diskName != "" && strings.EqualFold(*disk.Name, diskName)) ||
|
||||||
(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && strings.EqualFold(*disk.Vhd.URI, diskURI)) ||
|
(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && strings.EqualFold(*disk.Vhd.URI, diskURI)) ||
|
||||||
(disk.ManagedDisk != nil && strings.EqualFold(*disk.ManagedDisk.ID, diskURI)) {
|
(disk.ManagedDisk != nil && strings.EqualFold(*disk.ManagedDisk.ID, diskURI)) {
|
||||||
// found the disk
|
if disk.ToBeDetached != nil && *disk.ToBeDetached {
|
||||||
klog.V(2).Infof("azureDisk - find disk: lun %d name %q uri %q", *disk.Lun, diskName, diskURI)
|
klog.Warningf("azureDisk - find disk(ToBeDetached): lun %d name %q uri %q", *disk.Lun, diskName, diskURI)
|
||||||
return *disk.Lun, nil
|
} else {
|
||||||
|
// found the disk
|
||||||
|
klog.V(2).Infof("azureDisk - find disk: lun %d name %q uri %q", *disk.Lun, diskName, diskURI)
|
||||||
|
return *disk.Lun, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return -1, fmt.Errorf("cannot find Lun for disk %s", diskName)
|
return -1, fmt.Errorf("cannot find Lun for disk %s", diskName)
|
||||||
|
@ -201,6 +201,10 @@ func (c *ManagedDiskController) DeleteManagedDisk(diskURI string) error {
|
|||||||
ctx, cancel := getContextWithCancel()
|
ctx, cancel := getContextWithCancel()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
|
if _, ok := c.common.diskAttachDetachMap.Load(strings.ToLower(diskURI)); ok {
|
||||||
|
return fmt.Errorf("failed to delete disk(%s) since it's in attaching or detaching state", diskURI)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = c.common.cloud.DisksClient.Delete(ctx, resourceGroup, diskName)
|
_, err = c.common.cloud.DisksClient.Delete(ctx, resourceGroup, diskName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
Loading…
Reference in New Issue
Block a user