diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index e6d953967cb..6eb9f3dc024 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -1691,6 +1691,20 @@ func newAWSDisk(aws *Cloud, name KubernetesVolumeID) (*awsDisk, error) { return disk, nil } +// Helper function for describeVolume callers. Tries to retype given error to AWS error +// and returns true in case the AWS error is "InvalidVolume.NotFound", false otherwise +func isAWSErrorVolumeNotFound(err error) bool { + if err != nil { + if awsError, ok := err.(awserr.Error); ok { + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html + if awsError.Code() == "InvalidVolume.NotFound" { + return true + } + } + } + return false +} + // Gets the full information about this volume from the EC2 API func (d *awsDisk) describeVolume() (*ec2.Volume, error) { volumeID := d.awsID @@ -1701,13 +1715,13 @@ func (d *awsDisk) describeVolume() (*ec2.Volume, error) { volumes, err := d.ec2.DescribeVolumes(request) if err != nil { - return nil, fmt.Errorf("error querying ec2 for volume %q: %q", volumeID, err) + return nil, err } if len(volumes) == 0 { - return nil, fmt.Errorf("no volumes found for volume %q", volumeID) + return nil, fmt.Errorf("no volumes found") } if len(volumes) > 1 { - return nil, fmt.Errorf("multiple volumes found for volume %q", volumeID) + return nil, fmt.Errorf("multiple volumes found") } return volumes[0], nil } @@ -1811,12 +1825,29 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, err := wait.ExponentialBackoff(backoff, func() (bool, error) { info, err := d.describeVolume() if err != nil { + // The VolumeNotFound error is special -- we don't need to wait for it to repeat + if isAWSErrorVolumeNotFound(err) { + if status == "detached" { + // The disk doesn't exist, assume it's detached, log warning and stop waiting + glog.Warningf("Waiting for volume %q to be detached but the volume does not exist", d.awsID) + stateStr := "detached" + attachment = &ec2.VolumeAttachment{ + State: &stateStr, + } + return true, nil + } + if status == "attached" { + // The disk doesn't exist, complain, give up waiting and report error + glog.Warningf("Waiting for volume %q to be attached but the volume does not exist", d.awsID) + return false, err + } + } describeErrorCount++ if describeErrorCount > volumeAttachmentStatusConsecutiveErrorLimit { // report the error return false, err } else { - glog.Warningf("Ignoring error from describe volume; will retry: %q", err) + glog.Warningf("Ignoring error from describe volume for volume %q; will retry: %q", d.awsID, err) return false, nil } } else { @@ -2044,7 +2075,13 @@ func (c *Cloud) AttachDisk(diskName KubernetesVolumeID, nodeName types.NodeName, func (c *Cloud) DetachDisk(diskName KubernetesVolumeID, nodeName types.NodeName) (string, error) { diskInfo, attached, err := c.checkIfAttachedToNode(diskName, nodeName) if err != nil { - return "", err + if isAWSErrorVolumeNotFound(err) { + // Someone deleted the volume being detached; complain, but do nothing else and return success + glog.Warningf("DetachDisk %s called for node %s but volume does not exist; assuming the volume is detached", diskName, nodeName) + return "", nil + } else { + return "", err + } } if !attached && diskInfo.ec2Instance != nil { @@ -2339,9 +2376,15 @@ func (c *Cloud) GetDiskPath(volumeName KubernetesVolumeID) (string, error) { // DiskIsAttached implements Volumes.DiskIsAttached func (c *Cloud) DiskIsAttached(diskName KubernetesVolumeID, nodeName types.NodeName) (bool, error) { - diskInfo, attached, err := c.checkIfAttachedToNode(diskName, nodeName) - if diskInfo == nil { - return true, err + _, attached, err := c.checkIfAttachedToNode(diskName, nodeName) + if err != nil { + if isAWSErrorVolumeNotFound(err) { + // The disk doesn't exist, can't be attached + glog.Warningf("DiskIsAttached called for volume %s on node %s but the volume does not exist", diskName, nodeName) + return false, nil + } else { + return true, err + } } return attached, nil diff --git a/pkg/cloudprovider/providers/aws/volumes.go b/pkg/cloudprovider/providers/aws/volumes.go index 2ac932d43d7..48aaded947c 100644 --- a/pkg/cloudprovider/providers/aws/volumes.go +++ b/pkg/cloudprovider/providers/aws/volumes.go @@ -119,10 +119,9 @@ func (c *Cloud) checkIfAttachedToNode(diskName KubernetesVolumeID, nodeName type info, err := disk.describeVolume() if err != nil { - describeError := fmt.Errorf("Error describing volume %s with %v", diskName, err) - glog.Warning(describeError) + glog.Warning("Error describing volume %s with %v", diskName, err) awsDiskInfo.volumeState = "unknown" - return awsDiskInfo, false, describeError + return awsDiskInfo, false, err } awsDiskInfo.volumeState = aws.StringValue(info.State)