From f99cb99a1b9543a89c99dbf147cc62415e1f641d Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 15 Sep 2025 13:16:22 -0400 Subject: [PATCH] Wait for quota to report used before creating pvc This should avoid races between pvc creation and quota creation --- test/e2e/storage/csimock/base.go | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/e2e/storage/csimock/base.go b/test/e2e/storage/csimock/base.go index 4287184b722..4c51a11a319 100644 --- a/test/e2e/storage/csimock/base.go +++ b/test/e2e/storage/csimock/base.go @@ -28,10 +28,13 @@ import ( csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega/gcustom" + gomegatypes "github.com/onsi/gomega/types" "google.golang.org/grpc/codes" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" @@ -518,9 +521,46 @@ func (m *mockDriverSetup) createResourceQuota(ctx context.Context, quota *v1.Res quota, err = f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Create(ctx, quota, metav1.CreateOptions{}) framework.ExpectNoError(err, "Failed to create resourceQuota") m.quotas = append(m.quotas, quota) + usedResources := v1.ResourceList{} + usedResources[pvcSizeQuotaKey] = resource.MustParse("0") + usedResources[pvcCountQuotaKey] = resource.MustParse("0") + err = m.waitForResourceQuota(ctx, f.Namespace.Name, quota.Name, usedResources) + framework.ExpectNoError(err, "Failed to wait for resourcequota creation") return quota } +func (m *mockDriverSetup) waitForResourceQuota(ctx context.Context, ns, quotaName string, used v1.ResourceList) error { + var lastResourceQuota *v1.ResourceQuota + f := m.f + err := framework.Gomega().Eventually(ctx, framework.GetObject(f.ClientSet.CoreV1().ResourceQuotas(ns).Get, quotaName, metav1.GetOptions{})).Should(haveUsedResources(used, &lastResourceQuota)) + if lastResourceQuota != nil && err == nil { + framework.Logf("Got expected ResourceQuota:\n%s", format.Object(lastResourceQuota, 1)) + } + return err +} + +func haveUsedResources(used v1.ResourceList, lastResourceQuota **v1.ResourceQuota) gomegatypes.GomegaMatcher { + // The template emits the actual ResourceQuota object as YAML. + // In particular the ManagedFields are interesting because both + // kube-apiserver and kube-controller-manager set the status. + return gcustom.MakeMatcher(func(resourceQuota *v1.ResourceQuota) (bool, error) { + if lastResourceQuota != nil { + *lastResourceQuota = resourceQuota + } + // used may not yet be calculated + if resourceQuota.Status.Used == nil { + return false, nil + } + // verify that the quota shows the expected used resource values + for k, v := range used { + if actualValue, found := resourceQuota.Status.Used[k]; !found || (actualValue.Cmp(v) != 0) { + return false, nil + } + } + return true, nil + }).WithTemplate("Expected:\n{{.FormattedActual}}\n{{.To}} have the following .status.used entries:\n{{range $key, $value := .Data}} {{$key}}: \"{{$value.ToUnstructured}}\"\n{{end}}").WithTemplateData(used /* Formatting of the map is done inside the template. */) +} + func waitForCSIDriver(cs clientset.Interface, driverName string) error { timeout := 4 * time.Minute