mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 04:33:26 +00:00
Merge pull request #99664 from AndyXiangLi/aws-delete-on-creation-fail
delete leaked volume if driver don't know the volume status -- aws
This commit is contained in:
commit
b0fb5264e8
@ -2555,7 +2555,7 @@ func (c *Cloud) CreateDisk(volumeOptions *VolumeOptions) (KubernetesVolumeID, er
|
|||||||
createType = DefaultVolumeType
|
createType = DefaultVolumeType
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return "", fmt.Errorf("invalid AWS VolumeType %q", volumeOptions.VolumeType)
|
return KubernetesVolumeID(""), fmt.Errorf("invalid AWS VolumeType %q", volumeOptions.VolumeType)
|
||||||
}
|
}
|
||||||
|
|
||||||
request := &ec2.CreateVolumeInput{}
|
request := &ec2.CreateVolumeInput{}
|
||||||
@ -2587,12 +2587,12 @@ func (c *Cloud) CreateDisk(volumeOptions *VolumeOptions) (KubernetesVolumeID, er
|
|||||||
|
|
||||||
response, err := c.ec2.CreateVolume(request)
|
response, err := c.ec2.CreateVolume(request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return KubernetesVolumeID(""), err
|
||||||
}
|
}
|
||||||
|
|
||||||
awsID := EBSVolumeID(aws.StringValue(response.VolumeId))
|
awsID := EBSVolumeID(aws.StringValue(response.VolumeId))
|
||||||
if awsID == "" {
|
if awsID == "" {
|
||||||
return "", fmt.Errorf("VolumeID was not returned by CreateVolume")
|
return KubernetesVolumeID(""), fmt.Errorf("VolumeID was not returned by CreateVolume")
|
||||||
}
|
}
|
||||||
volumeName := KubernetesVolumeID("aws://" + aws.StringValue(response.AvailabilityZone) + "/" + string(awsID))
|
volumeName := KubernetesVolumeID("aws://" + aws.StringValue(response.AvailabilityZone) + "/" + string(awsID))
|
||||||
|
|
||||||
@ -2605,8 +2605,22 @@ func (c *Cloud) CreateDisk(volumeOptions *VolumeOptions) (KubernetesVolumeID, er
|
|||||||
// because Kubernetes may have limited permissions to the key.
|
// because Kubernetes may have limited permissions to the key.
|
||||||
if isAWSErrorVolumeNotFound(err) {
|
if isAWSErrorVolumeNotFound(err) {
|
||||||
err = fmt.Errorf("failed to create encrypted volume: the volume disappeared after creation, most likely due to inaccessible KMS encryption key")
|
err = fmt.Errorf("failed to create encrypted volume: the volume disappeared after creation, most likely due to inaccessible KMS encryption key")
|
||||||
|
} else {
|
||||||
|
// When DescribeVolumes api failed, plugin will lose track on the volumes' state
|
||||||
|
// driver should be able to clean up these kind of volumes to make sure they are not leaked on customers' account
|
||||||
|
klog.V(5).Infof("Failed to create the volume %v due to %v. Will try to delete it.", volumeName, err)
|
||||||
|
awsDisk, newDiskError := newAWSDisk(c, volumeName)
|
||||||
|
if newDiskError != nil {
|
||||||
|
klog.Errorf("Failed to delete the volume %v due to error: %v", volumeName, newDiskError)
|
||||||
|
} else {
|
||||||
|
if _, deleteVolumeError := awsDisk.deleteVolume(); deleteVolumeError != nil {
|
||||||
|
klog.Errorf("Failed to delete the volume %v due to error: %v", volumeName, deleteVolumeError)
|
||||||
|
} else {
|
||||||
|
klog.V(5).Infof("%v is deleted because it is not in desired state after waiting", volumeName)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return "", err
|
return KubernetesVolumeID(""), err
|
||||||
}
|
}
|
||||||
|
|
||||||
return volumeName, nil
|
return volumeName, nil
|
||||||
|
@ -73,6 +73,11 @@ func (m *MockedFakeEC2) DescribeVolumes(request *ec2.DescribeVolumesInput) ([]*e
|
|||||||
return args.Get(0).([]*ec2.Volume), nil
|
return args.Get(0).([]*ec2.Volume), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *MockedFakeEC2) DeleteVolume(request *ec2.DeleteVolumeInput) (*ec2.DeleteVolumeOutput, error) {
|
||||||
|
args := m.Called(request)
|
||||||
|
return args.Get(0).(*ec2.DeleteVolumeOutput), nil
|
||||||
|
}
|
||||||
|
|
||||||
func (m *MockedFakeEC2) DescribeSecurityGroups(request *ec2.DescribeSecurityGroupsInput) ([]*ec2.SecurityGroup, error) {
|
func (m *MockedFakeEC2) DescribeSecurityGroups(request *ec2.DescribeSecurityGroupsInput) ([]*ec2.SecurityGroup, error) {
|
||||||
args := m.Called(request)
|
args := m.Called(request)
|
||||||
return args.Get(0).([]*ec2.SecurityGroup), nil
|
return args.Get(0).([]*ec2.SecurityGroup), nil
|
||||||
@ -2363,6 +2368,51 @@ func TestCreateDisk(t *testing.T) {
|
|||||||
awsServices.ec2.(*MockedFakeEC2).AssertExpectations(t)
|
awsServices.ec2.(*MockedFakeEC2).AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCreateDiskFailDescribeVolume(t *testing.T) {
|
||||||
|
awsServices := newMockedFakeAWSServices(TestClusterID)
|
||||||
|
c, _ := newAWSCloud(CloudConfig{}, awsServices)
|
||||||
|
|
||||||
|
volumeOptions := &VolumeOptions{
|
||||||
|
AvailabilityZone: "us-east-1a",
|
||||||
|
CapacityGB: 10,
|
||||||
|
}
|
||||||
|
request := &ec2.CreateVolumeInput{
|
||||||
|
AvailabilityZone: aws.String("us-east-1a"),
|
||||||
|
Encrypted: aws.Bool(false),
|
||||||
|
VolumeType: aws.String(DefaultVolumeType),
|
||||||
|
Size: aws.Int64(10),
|
||||||
|
TagSpecifications: []*ec2.TagSpecification{
|
||||||
|
{ResourceType: aws.String(ec2.ResourceTypeVolume), Tags: []*ec2.Tag{
|
||||||
|
// CreateVolume from MockedFakeEC2 expects sorted tags, so we need to
|
||||||
|
// always have these tags sorted:
|
||||||
|
{Key: aws.String(TagNameKubernetesClusterLegacy), Value: aws.String(TestClusterID)},
|
||||||
|
{Key: aws.String(fmt.Sprintf("%s%s", TagNameKubernetesClusterPrefix, TestClusterID)), Value: aws.String(ResourceLifecycleOwned)},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
volume := &ec2.Volume{
|
||||||
|
AvailabilityZone: aws.String("us-east-1a"),
|
||||||
|
VolumeId: aws.String("vol-volumeId0"),
|
||||||
|
State: aws.String("creating"),
|
||||||
|
}
|
||||||
|
awsServices.ec2.(*MockedFakeEC2).On("CreateVolume", request).Return(volume, nil)
|
||||||
|
|
||||||
|
describeVolumesRequest := &ec2.DescribeVolumesInput{
|
||||||
|
VolumeIds: []*string{aws.String("vol-volumeId0")},
|
||||||
|
}
|
||||||
|
deleteVolumeRequest := &ec2.DeleteVolumeInput{
|
||||||
|
VolumeId: aws.String("vol-volumeId0"),
|
||||||
|
}
|
||||||
|
awsServices.ec2.(*MockedFakeEC2).On("DescribeVolumes", describeVolumesRequest).Return([]*ec2.Volume{volume}, nil)
|
||||||
|
awsServices.ec2.(*MockedFakeEC2).On("DeleteVolume", deleteVolumeRequest).Return(&ec2.DeleteVolumeOutput{}, nil)
|
||||||
|
|
||||||
|
volumeID, err := c.CreateDisk(volumeOptions)
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Equal(t, volumeID, KubernetesVolumeID(""))
|
||||||
|
awsServices.ec2.(*MockedFakeEC2).AssertExpectations(t)
|
||||||
|
}
|
||||||
|
|
||||||
func TestRegionIsValid(t *testing.T) {
|
func TestRegionIsValid(t *testing.T) {
|
||||||
fake := newMockedFakeAWSServices("fakeCluster")
|
fake := newMockedFakeAWSServices("fakeCluster")
|
||||||
fake.selfInstance.Placement = &ec2.Placement{
|
fake.selfInstance.Placement = &ec2.Placement{
|
||||||
|
Loading…
Reference in New Issue
Block a user