diff --git a/test/e2e/framework/volume/fixtures.go b/test/e2e/framework/volume/fixtures.go index ecc59b63da1..b2938eae896 100644 --- a/test/e2e/framework/volume/fixtures.go +++ b/test/e2e/framework/volume/fixtures.go @@ -256,6 +256,24 @@ func getVolumeHandle(ctx context.Context, cs clientset.Interface, claimName stri return pv.Spec.CSI.VolumeHandle } +// WaitForVolumeAttachmentCreated waits for the VolumeAttachment with the passed in attachmentName to be created. +func WaitForVolumeAttachmentCreated(ctx context.Context, attachmentName string, cs clientset.Interface, interval, timeout time.Duration) error { + waitErr := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) { + _, err := cs.StorageV1().VolumeAttachments().Get(ctx, attachmentName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil + }) + if waitErr != nil { + return fmt.Errorf("error waiting volume attachment %v to be created: %w", attachmentName, waitErr) + } + return nil +} + // WaitForVolumeAttachmentTerminated waits for the VolumeAttachment with the passed in attachmentName to be terminated. func WaitForVolumeAttachmentTerminated(ctx context.Context, attachmentName string, cs clientset.Interface, timeout time.Duration) error { waitErr := wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { diff --git a/test/e2e/storage/csimock/base.go b/test/e2e/storage/csimock/base.go index 35854b7f678..985369aa61a 100644 --- a/test/e2e/storage/csimock/base.go +++ b/test/e2e/storage/csimock/base.go @@ -59,6 +59,7 @@ const ( csiPodUnschedulableTimeout = 5 * time.Minute csiResizeWaitPeriod = 5 * time.Minute csiVolumeAttachmentTimeout = 7 * time.Minute + csiDriverTimeout = 2 * time.Minute // how long to wait for GetVolumeStats csiNodeVolumeStatWaitPeriod = 2 * time.Minute // how long to wait for Resizing Condition on PVC to appear @@ -1247,3 +1248,20 @@ func waitForMaxVolumeCondition(pod *v1.Pod, cs clientset.Interface) error { } return nil } + +func waitForCSIDriverDeleted(ctx context.Context, cs clientset.Interface, driverName string, interval, timeout time.Duration) error { + waitErr := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) { + _, err := cs.StorageV1().CSIDrivers().Get(ctx, driverName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil + }) + if waitErr != nil { + return fmt.Errorf("error waiting for CSIDriver %s to be deleted: %w", driverName, waitErr) + } + return nil +} diff --git a/test/e2e/storage/csimock/csi_attach_volume.go b/test/e2e/storage/csimock/csi_attach_volume.go index d343da6e932..9778a086747 100644 --- a/test/e2e/storage/csimock/csi_attach_volume.go +++ b/test/e2e/storage/csimock/csi_attach_volume.go @@ -25,6 +25,7 @@ import ( "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -108,6 +109,58 @@ var _ = utils.SIGDescribe("CSI Mock volume attach", func() { } }) + ginkgo.Context("When CSIDriver AttachRequired changes from true to false", func() { + f.It("should cleanup VolumeAttachment properly", f.WithSlow(), func(ctx context.Context) { + var err error + m.init(ctx, testParameters{registerDriver: true, disableAttach: false}) + ginkgo.DeferCleanup(m.cleanup) + + driverName := m.config.GetUniqueDriverName() + csiDriver, err := m.cs.StorageV1().CSIDrivers().Get(ctx, driverName, metav1.GetOptions{}) + framework.ExpectNoError(err, "Failed to get CSIDriver: %v", err) + driver := csiDriver.DeepCopy() + + _, claim, pod := m.createPod(ctx, pvcReference) + if pod == nil { + return + } + testConfig := storageframework.ConvertTestConfig(m.config) + attachmentName := e2evolume.GetVolumeAttachmentName(ctx, m.cs, testConfig, m.provisioner, claim.Name, claim.Namespace) + ginkgo.DeferCleanup(framework.IgnoreNotFound(m.cs.StorageV1().VolumeAttachments().Delete), attachmentName, metav1.DeleteOptions{}) + + // Use a short interval to quickly detect VolumeAttachment creation and immediately recreate the CSIDriver + interval := 200 * time.Millisecond + ginkgo.By("Checking if VolumeAttachment was created for the pod") + err = e2evolume.WaitForVolumeAttachmentCreated(ctx, attachmentName, m.cs, interval, csiVolumeAttachmentTimeout) + framework.ExpectNoError(err, "Failed to wait for VolumeAttachment %s to be created: %v", attachmentName, err) + ginkgo.By("Delete CSIDriver object") + err = m.cs.StorageV1().CSIDrivers().Delete(ctx, driverName, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Failed to delete CSIDriver: %v", err) + err = waitForCSIDriverDeleted(ctx, m.cs, driverName, interval, csiDriverTimeout) + framework.ExpectNoError(err, "Failed to delete CSIDriver: %v", err) + + attachRequired := false + driver.Spec.AttachRequired = &attachRequired + newDriver := &storagev1.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: driverName, + }, + Spec: driver.Spec, + } + ginkgo.By("Recreating CSIDriver with AttachRequired=false") + _, err = m.cs.StorageV1().CSIDrivers().Create(ctx, newDriver, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create CSIDriver: %v", err) + err = waitForCSIDriver(m.cs, driverName) + framework.ExpectNoError(err, "Failed to get CSIDriver: %v", err) + + ginkgo.By("Wait for the volumeattachment to be deleted") + err = e2evolume.WaitForVolumeAttachmentTerminated(ctx, attachmentName, m.cs, csiVolumeAttachmentTimeout) + framework.ExpectNoError(err, "Failed to delete VolumeAttachment: %v", err) + err = e2epod.WaitForPodNameRunningInNamespace(ctx, m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + }) + }) + ginkgo.Context("CSI CSIDriver deployment after pod creation using non-attachable mock driver", func() { f.It("should bringup pod after deploying CSIDriver attach=false", f.WithSlow(), func(ctx context.Context) { var err error