diff --git a/test/e2e/storage/csimock/base.go b/test/e2e/storage/csimock/base.go index d58a7b33c57..a1f318db340 100644 --- a/test/e2e/storage/csimock/base.go +++ b/test/e2e/storage/csimock/base.go @@ -116,6 +116,7 @@ type mockDriverSetup struct { pods []*v1.Pod pvcs []*v1.PersistentVolumeClaim pvs []*v1.PersistentVolume + quotas []*v1.ResourceQuota sc map[string]*storagev1.StorageClass vsc map[string]*unstructured.Unstructured driver drivers.MockCSITestDriver @@ -259,6 +260,15 @@ func (m *mockDriverSetup) cleanup(ctx context.Context) { m.config.Framework.DynamicClient.Resource(utils.SnapshotClassGVR).Delete(context.TODO(), vsc.GetName(), metav1.DeleteOptions{}) } + for _, quota := range m.quotas { + ginkgo.By(fmt.Sprintf("Deleting quota %s", quota.Name)) + if err := cs.CoreV1().ResourceQuotas(quota.Namespace).Delete(ctx, quota.Name, metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, err) + } + } + } + err := utilerrors.NewAggregate(errs) framework.ExpectNoError(err, "while cleaning up after test") } @@ -497,6 +507,22 @@ func (m *mockDriverSetup) createPodWithSELinux(ctx context.Context, accessModes return class, claim, pod } +func (m *mockDriverSetup) createResourceQuota(ctx context.Context, quota *v1.ResourceQuota) (*v1.ResourceQuota, error) { + ginkgo.By("Creating Resource Quota") + f := m.f + + quota.ObjectMeta = metav1.ObjectMeta{ + Name: f.UniqueName + "-quota", + Namespace: f.Namespace.Name, + } + var err error + + quota, err = f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Create(ctx, quota, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create resourceQuota") + m.quotas = append(m.quotas, quota) + return quota, err +} + func waitForCSIDriver(cs clientset.Interface, driverName string) error { timeout := 4 * time.Minute diff --git a/test/e2e/storage/csimock/csi_volume_expansion.go b/test/e2e/storage/csimock/csi_volume_expansion.go index d2e52e69548..51688d300cc 100644 --- a/test/e2e/storage/csimock/csi_volume_expansion.go +++ b/test/e2e/storage/csimock/csi_volume_expansion.go @@ -32,8 +32,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/storage/drivers" @@ -55,6 +53,8 @@ const ( const ( resizePollInterval = 2 * time.Second + pvcCountQuotaKey = "persistentvolumeclaims" + pvcSizeQuotaKey = "requests.storage" ) var ( @@ -71,6 +71,8 @@ type recoveryTest struct { disableControllerExpansion bool expectedResizeStatus v1.ClaimResourceStatus recoverySize resource.Quantity + fullResourceQuota *v1.ResourceQuota + expectedQuotaUsage *v1.ResourceQuota } var _ = utils.SIGDescribe("CSI Mock volume expansion", func() { @@ -396,7 +398,7 @@ var _ = utils.SIGDescribe("CSI Mock volume expansion", func() { } }) - f.Context("Expansion with recovery", feature.RecoverVolumeExpansionFailure, framework.WithFeatureGate(features.RecoverVolumeExpansionFailure), func() { + f.Context("Expansion with recovery", func() { tests := []recoveryTest{ { name: "should record target size in allocated resources", @@ -405,6 +407,22 @@ var _ = utils.SIGDescribe("CSI Mock volume expansion", func() { disableControllerExpansion: false, simulatedCSIDriverError: expansionSuccess, expectedResizeStatus: "", + fullResourceQuota: &v1.ResourceQuota{ + Spec: v1.ResourceQuotaSpec{ + Hard: v1.ResourceList{ + pvcSizeQuotaKey: resource.MustParse("20Gi"), + pvcCountQuotaKey: resource.MustParse("5"), + }, + }, + }, + expectedQuotaUsage: &v1.ResourceQuota{ + Status: v1.ResourceQuotaStatus{ + Used: v1.ResourceList{ + pvcSizeQuotaKey: resource.MustParse("4Gi"), + pvcCountQuotaKey: resource.MustParse("1"), + }, + }, + }, }, { name: "should allow recovery if controller expansion fails with infeasible error", @@ -466,6 +484,10 @@ var _ = utils.SIGDescribe("CSI Mock volume expansion", func() { m.init(ctx, params) ginkgo.DeferCleanup(m.cleanup) + if test.fullResourceQuota != nil { + m.createResourceQuota(ctx, test.fullResourceQuota) + } + sc, pvc, pod := m.createPod(ctx, pvcReference) gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod for resizing") @@ -493,12 +515,46 @@ var _ = utils.SIGDescribe("CSI Mock volume expansion", func() { } else { validateRecoveryBehaviour(ctx, pvc, m, test) } + + if test.expectedQuotaUsage != nil { + validateQuotaUsage(ctx, m, test.expectedQuotaUsage) + } }) } - }) }) +func validateQuotaUsage(ctx context.Context, m *mockDriverSetup, expectedQuota *v1.ResourceQuota) { + ginkgo.By("Waiting for resource quota usage to be updated") + var err error + var quota *v1.ResourceQuota + var usedCount resource.Quantity + var usedSize resource.Quantity + + expectedCount := expectedQuota.Status.Used[pvcCountQuotaKey] + expectedUsedSize := expectedQuota.Status.Used[pvcSizeQuotaKey] + + waitErr := wait.PollUntilContextTimeout(ctx, resizePollInterval, csiResizeWaitPeriod, true, func(pollContext context.Context) (bool, error) { + quota, err = m.cs.CoreV1().ResourceQuotas(expectedQuota.Namespace).Get(pollContext, expectedQuota.Name, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("error fetching resource quota %q: %w", expectedQuota.Name, err) + } + if quota.Status.Used == nil { + return false, nil + } + usedCount = quota.Status.Used[pvcCountQuotaKey] + usedSize = quota.Status.Used[pvcSizeQuotaKey] + if usedCount.Cmp(expectedCount) == 0 && usedSize.Cmp(expectedUsedSize) == 0 { + return true, nil + } + return false, nil + }) + + if waitErr != nil { + framework.Failf("error while waiting for resource quota usage to be updated, currentlyUsed: %s/%s, expected: %s/%s: %v", usedCount.String(), usedSize.String(), expectedCount.String(), expectedUsedSize.String(), waitErr) + } +} + func validateRecoveryBehaviour(ctx context.Context, pvc *v1.PersistentVolumeClaim, m *mockDriverSetup, test recoveryTest) { var err error ginkgo.By("Waiting for resizer to set allocated resource") diff --git a/test/e2e/storage/testsuites/volume_expand.go b/test/e2e/storage/testsuites/volume_expand.go index 3a56646b255..05b21b7da75 100644 --- a/test/e2e/storage/testsuites/volume_expand.go +++ b/test/e2e/storage/testsuites/volume_expand.go @@ -238,6 +238,9 @@ func (v *volumeExpandTestSuite) DefineTests(driver storageframework.TestDriver, framework.ExpectNoError(err, "While waiting for pvc to have fs resizing condition") l.resource.Pvc = npvc + err = VerifyAllocatedResources(l.resource.Pvc, v1.PersistentVolumeClaimNodeResizePending, pvcSize) + framework.ExpectNoError(err, "While verifying allocatedResources on PVC") + ginkgo.By("Creating a new pod with same volume") podConfig = e2epod.Config{ NS: f.Namespace.Name, @@ -561,6 +564,19 @@ func WaitForFSResize(ctx context.Context, pvc *v1.PersistentVolumeClaim, c clien return updatedPVC, nil } +func VerifyAllocatedResources(pvc *v1.PersistentVolumeClaim, resizeStatus v1.ClaimResourceStatus, allocatedSize resource.Quantity) error { + actualResizeStatus := pvc.Status.AllocatedResourceStatuses[v1.ResourceStorage] + if actualResizeStatus != resizeStatus { + return fmt.Errorf("pvc %q had %s resize status, expected %s", pvc.Name, actualResizeStatus, resizeStatus) + } + + actualAllocatedSize := pvc.Status.AllocatedResources.Storage() + if actualAllocatedSize.Cmp(allocatedSize) < 0 { + return fmt.Errorf("pvc %q had %s allocated size, expected %s", pvc.Name, actualAllocatedSize.String(), allocatedSize.String()) + } + return nil +} + func VerifyRecoveryRelatedFields(pvc *v1.PersistentVolumeClaim) error { resizeStatus := pvc.Status.AllocatedResourceStatuses[v1.ResourceStorage] if resizeStatus != "" {