diff --git a/test/e2e/framework/pv/wait.go b/test/e2e/framework/pv/wait.go index 6f903d996c6..ebfe227afe3 100644 --- a/test/e2e/framework/pv/wait.go +++ b/test/e2e/framework/pv/wait.go @@ -68,3 +68,42 @@ func WaitForPersistentVolumeClaimModified(ctx context.Context, c clientset.Inter }, nil })) } + +// WaitForPersistentVolumeClaimModificationFailure waits the given timeout duration for the specified claim to have +// failed to bind to the invalid volume attributes class. +// Returns an error if timeout occurs first. +func WaitForPersistentVolumeClaimModificationFailure(ctx context.Context, c clientset.Interface, claim *v1.PersistentVolumeClaim, timeout time.Duration) error { + desiredClass := ptr.Deref(claim.Spec.VolumeAttributesClassName, "") + + var match = func(claim *v1.PersistentVolumeClaim) bool { + for _, condition := range claim.Status.Conditions { + if condition.Type != v1.PersistentVolumeClaimVolumeModifyVolumeError { + return false + } + } + + // check if claim's current volume attributes class is NOT desired one, and has appropriate ModifyVolumeStatus + currentClass := ptr.Deref(claim.Status.CurrentVolumeAttributesClassName, "") + return claim.Status.Phase == v1.ClaimBound && + desiredClass != currentClass && claim.Status.ModifyVolumeStatus != nil && + (claim.Status.ModifyVolumeStatus.Status == v1.PersistentVolumeClaimModifyVolumeInProgress || + claim.Status.ModifyVolumeStatus.Status == v1.PersistentVolumeClaimModifyVolumeInfeasible) + } + + if match(claim) { + return nil + } + + return framework.Gomega(). + Eventually(ctx, framework.GetObject(c.CoreV1().PersistentVolumeClaims(claim.Namespace).Get, claim.Name, metav1.GetOptions{})). + WithTimeout(timeout). + Should(framework.MakeMatcher(func(claim *v1.PersistentVolumeClaim) (func() string, error) { + if match(claim) { + return nil, nil + } + + return func() string { + return fmt.Sprintf("expected claim's status to NOT be modified with the given VolumeAttirbutesClass %s, got instead:\n%s", desiredClass, format.Object(claim, 1)) + }, nil + })) +} diff --git a/test/e2e/storage/testsuites/volume_modify.go b/test/e2e/storage/testsuites/volume_modify.go index 11321f6d030..e8745fa728d 100644 --- a/test/e2e/storage/testsuites/volume_modify.go +++ b/test/e2e/storage/testsuites/volume_modify.go @@ -41,11 +41,10 @@ import ( ) const ( - modifyPollInterval = 2 * time.Second - setVACWaitPeriod = 30 * time.Second - modifyingConditionSyncWaitPeriod = 2 * time.Minute - modifyVolumeWaitPeriod = 10 * time.Minute - vacCleanupWaitPeriod = 30 * time.Second + modifyPollInterval = 2 * time.Second + setVACWaitPeriod = 30 * time.Second + modifyVolumeWaitPeriod = 10 * time.Minute + vacCleanupWaitPeriod = 30 * time.Second ) type volumeModifyTestSuite struct { @@ -227,6 +226,46 @@ func (v *volumeModifyTestSuite) DefineTests(driver storageframework.TestDriver, err = e2epv.WaitForPersistentVolumeClaimModified(ctx, f.ClientSet, l.resource.Pvc, modifyVolumeWaitPeriod) framework.ExpectNoError(err, "While waiting for PVC to have expected VAC") }) + + ginkgo.It("should recover from invalid target VAC by updating PVC to new valid VAC", func(ctx context.Context) { + init(ctx, false /* volume created without VAC */) + ginkgo.DeferCleanup(cleanup) + + // Create VAC with unsupported parameter + invalidVAC := MakeInvalidVAC(l.config) + _, err := f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Create(ctx, invalidVAC, metav1.CreateOptions{}) + framework.ExpectNoError(err, "While creating new VolumeAttributesClass") + ginkgo.DeferCleanup(CleanupVAC, invalidVAC, f.ClientSet, vacCleanupWaitPeriod) + + ginkgo.By("Creating a pod with dynamically provisioned volume") + podConfig := e2epod.Config{ + NS: f.Namespace.Name, + PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc}, + SeLinuxLabel: e2epod.GetLinuxLabel(), + NodeSelection: l.config.ClientNodeSelection, + ImageID: e2epod.GetDefaultTestImageID(), + } + pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart) + ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod) + framework.ExpectNoError(err, "While creating pod for modifying") + + ginkgo.By("Attempting to modify PVC via VolumeAttributeClass that contains unsupported parameters") + newPVC := SetPVCVACName(ctx, l.resource.Pvc, invalidVAC.Name, f.ClientSet, setVACWaitPeriod) + l.resource.Pvc = newPVC + gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil()) + + ginkgo.By("Waiting for modification to fail") + err = e2epv.WaitForPersistentVolumeClaimModificationFailure(ctx, f.ClientSet, l.resource.Pvc, modifyVolumeWaitPeriod) + framework.ExpectNoError(err, "While waiting for PVC to have conditions") + + ginkgo.By("Modifying PVC to new valid VAC") + l.resource.Pvc = SetPVCVACName(ctx, l.resource.Pvc, l.vac.Name, f.ClientSet, setVACWaitPeriod) + gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil()) + + ginkgo.By("Checking PVC status") + err = e2epv.WaitForPersistentVolumeClaimModified(ctx, f.ClientSet, l.resource.Pvc, modifyVolumeWaitPeriod) + framework.ExpectNoError(err, "While waiting for PVC to have expected VAC") + }) } // SetPVCVACName sets the VolumeAttributesClassName on a PVC object @@ -246,6 +285,15 @@ func SetPVCVACName(ctx context.Context, origPVC *v1.PersistentVolumeClaim, name return patchedPVC } +func MakeInvalidVAC(config *storageframework.PerTestConfig) *storagev1beta1.VolumeAttributesClass { + return storageframework.CopyVolumeAttributesClass(&storagev1beta1.VolumeAttributesClass{ + DriverName: config.GetUniqueDriverName(), + Parameters: map[string]string{ + "xxInvalidParameterKey": "xxInvalidParameterValue", + }, + }, config.Framework.Namespace.Name, "e2e-vac-invalid") +} + func CleanupVAC(ctx context.Context, vac *storagev1beta1.VolumeAttributesClass, c clientset.Interface, timeout time.Duration) { gomega.Eventually(ctx, func() error { return c.StorageV1beta1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{})