Add e2e test for VolumeAttachment cleanup when CSIDriver AttachRequired changes

Signed-off-by: hongkang <mzhkcj50@gmail.com>

test/e2e/framework/volume/fixtures.go

@@ -256,6 +256,24 @@ func getVolumeHandle(ctx context.Context, cs clientset.Interface, claimName stri
 	return pv.Spec.CSI.VolumeHandle
 }
 
+// WaitForVolumeAttachmentCreated waits for the VolumeAttachment with the passed in attachmentName to be created.
+func WaitForVolumeAttachmentCreated(ctx context.Context, attachmentName string, cs clientset.Interface, interval, timeout time.Duration) error {
+	waitErr := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
+		_, err := cs.StorageV1().VolumeAttachments().Get(ctx, attachmentName, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				return false, nil
+			}
+			return false, err
+		}
+		return true, nil
+	})
+	if waitErr != nil {
+		return fmt.Errorf("error waiting volume attachment %v to be created: %w", attachmentName, waitErr)
+	}
+	return nil
+}
+
 // WaitForVolumeAttachmentTerminated waits for the VolumeAttachment with the passed in attachmentName to be terminated.
 func WaitForVolumeAttachmentTerminated(ctx context.Context, attachmentName string, cs clientset.Interface, timeout time.Duration) error {
 	waitErr := wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) {

test/e2e/storage/csimock/base.go

@@ -59,6 +59,7 @@ const (
 	csiPodUnschedulableTimeout = 5 * time.Minute
 	csiResizeWaitPeriod        = 5 * time.Minute
 	csiVolumeAttachmentTimeout = 7 * time.Minute
+	csiDriverTimeout           = 2 * time.Minute
 	// how long to wait for GetVolumeStats
 	csiNodeVolumeStatWaitPeriod = 2 * time.Minute
 	// how long to wait for Resizing Condition on PVC to appear
@@ -1247,3 +1248,20 @@ func waitForMaxVolumeCondition(pod *v1.Pod, cs clientset.Interface) error {
 	}
 	return nil
 }
+
+func waitForCSIDriverDeleted(ctx context.Context, cs clientset.Interface, driverName string, interval, timeout time.Duration) error {
+	waitErr := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
+		_, err := cs.StorageV1().CSIDrivers().Get(ctx, driverName, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	})
+	if waitErr != nil {
+		return fmt.Errorf("error waiting for CSIDriver %s to be deleted: %w", driverName, waitErr)
+	}
+	return nil
+}

test/e2e/storage/csimock/csi_attach_volume.go

@@ -25,6 +25,7 @@ import (
 	"github.com/onsi/gomega"
 
 	v1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -108,6 +109,58 @@ var _ = utils.SIGDescribe("CSI Mock volume attach", func() {
 		}
 	})
 
+	ginkgo.Context("When CSIDriver AttachRequired changes from true to false", func() {
+		f.It("should cleanup VolumeAttachment properly", f.WithSlow(), func(ctx context.Context) {
+			var err error
+			m.init(ctx, testParameters{registerDriver: true, disableAttach: false})
+			ginkgo.DeferCleanup(m.cleanup)
+
+			driverName := m.config.GetUniqueDriverName()
+			csiDriver, err := m.cs.StorageV1().CSIDrivers().Get(ctx, driverName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "Failed to get CSIDriver: %v", err)
+			driver := csiDriver.DeepCopy()
+
+			_, claim, pod := m.createPod(ctx, pvcReference)
+			if pod == nil {
+				return
+			}
+			testConfig := storageframework.ConvertTestConfig(m.config)
+			attachmentName := e2evolume.GetVolumeAttachmentName(ctx, m.cs, testConfig, m.provisioner, claim.Name, claim.Namespace)
+			ginkgo.DeferCleanup(framework.IgnoreNotFound(m.cs.StorageV1().VolumeAttachments().Delete), attachmentName, metav1.DeleteOptions{})
+
+			// Use a short interval to quickly detect VolumeAttachment creation and immediately recreate the CSIDriver
+			interval := 200 * time.Millisecond
+			ginkgo.By("Checking if VolumeAttachment was created for the pod")
+			err = e2evolume.WaitForVolumeAttachmentCreated(ctx, attachmentName, m.cs, interval, csiVolumeAttachmentTimeout)
+			framework.ExpectNoError(err, "Failed to wait for VolumeAttachment %s to be created: %v", attachmentName, err)
+			ginkgo.By("Delete CSIDriver object")
+			err = m.cs.StorageV1().CSIDrivers().Delete(ctx, driverName, metav1.DeleteOptions{})
+			framework.ExpectNoError(err, "Failed to delete CSIDriver: %v", err)
+			err = waitForCSIDriverDeleted(ctx, m.cs, driverName, interval, csiDriverTimeout)
+			framework.ExpectNoError(err, "Failed to delete CSIDriver: %v", err)
+
+			attachRequired := false
+			driver.Spec.AttachRequired = &attachRequired
+			newDriver := &storagev1.CSIDriver{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: driverName,
+				},
+				Spec: driver.Spec,
+			}
+			ginkgo.By("Recreating CSIDriver with AttachRequired=false")
+			_, err = m.cs.StorageV1().CSIDrivers().Create(ctx, newDriver, metav1.CreateOptions{})
+			framework.ExpectNoError(err, "Failed to create CSIDriver: %v", err)
+			err = waitForCSIDriver(m.cs, driverName)
+			framework.ExpectNoError(err, "Failed to get CSIDriver: %v", err)
+
+			ginkgo.By("Wait for the volumeattachment to be deleted")
+			err = e2evolume.WaitForVolumeAttachmentTerminated(ctx, attachmentName, m.cs, csiVolumeAttachmentTimeout)
+			framework.ExpectNoError(err, "Failed to delete VolumeAttachment: %v", err)
+			err = e2epod.WaitForPodNameRunningInNamespace(ctx, m.cs, pod.Name, pod.Namespace)
+			framework.ExpectNoError(err, "Failed to start pod: %v", err)
+		})
+	})
+
 	ginkgo.Context("CSI CSIDriver deployment after pod creation using non-attachable mock driver", func() {
 		f.It("should bringup pod after deploying CSIDriver attach=false", f.WithSlow(), func(ctx context.Context) {
 			var err error