diff --git a/test/e2e/storage/framework/volume_group_snapshot_resource.go b/test/e2e/storage/framework/volume_group_snapshot_resource.go index 1a7a180b12b..c977d533cea 100644 --- a/test/e2e/storage/framework/volume_group_snapshot_resource.go +++ b/test/e2e/storage/framework/volume_group_snapshot_resource.go @@ -22,8 +22,11 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -110,15 +113,149 @@ func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VolumeGroupSnapshott return gsclass, volumeGroupSnapshot, vgsc } -// CleanupResource deletes the VolumeGroupSnapshotClass and VolumeGroupSnapshot objects using a dynamic client. -func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeouts *framework.TimeoutContext) error { - defer ginkgo.GinkgoRecover() +// CleanupVGS deletes the VolumeGroupSnapshot and ensures the bound VolumeGroupSnapshotContent has Delete policy. +// It waits for the VolumeGroupSnapshot, its owned VolumeSnapshots, VolumeSnapshotContents, and the bound +// VolumeGroupSnapshotContent to be fully deleted to prevent resource leaks. +func (r *VolumeGroupSnapshotResource) CleanupVGS(ctx context.Context, timeouts *framework.TimeoutContext) error { + if r.VGS == nil { + return nil + } + + var cleanupErrs []error dc := r.Config.Framework.DynamicClient - err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, r.VGSClass.GetName(), metav1.DeleteOptions{}) - framework.ExpectNoError(err, "Failed to delete volume group snapshot class") + vgsNamespace := r.VGS.GetNamespace() + vgsName := r.VGS.GetName() + vgsUID := r.VGS.GetUID() + framework.Logf("deleting groupSnapshot %q/%q and ensuring content has Delete policy", vgsNamespace, vgsName) + + vgs, err := dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(vgsNamespace).Get(ctx, vgsName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // Hope that the underlying snapshot contents and resources are gone already + return nil + } + return fmt.Errorf("failed to get VGS %q: %w", vgsName, err) + } + r.VGS = vgs + + // Filter snapshots owned by this VGS + vss, err := dc.Resource(utils.SnapshotGVR).Namespace(vgsNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + + contentNamesSet := sets.NewString() + for _, vs := range vss.Items { + for _, owner := range vs.GetOwnerReferences() { + if owner.Kind == "VolumeGroupSnapshot" && owner.UID == vgsUID { + if status, ok := vs.Object["status"].(map[string]interface{}); ok { + if cName, ok := status["boundVolumeSnapshotContentName"].(string); ok && cName != "" { + contentNamesSet.Insert(cName) + } + } + } + } + } + + groupSnapshotStatus := r.VGS.Object["status"].(map[string]interface{}) + groupSnapshotContentName := groupSnapshotStatus["boundVolumeGroupSnapshotContentName"].(string) + framework.Logf("received groupSnapshotStatus %v", groupSnapshotStatus) + framework.Logf("groupSnapshotContentName %q", groupSnapshotContentName) + + boundVGSContent, err := dc.Resource(utils.VolumeGroupSnapshotContentGVR).Get(ctx, groupSnapshotContentName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get bound VGSContent %q: %w", boundVGSContent.GetName(), err) + } + + // Ensure deletion policy is set to Delete to prevent leaks + if boundVGSContent != nil { + spec := boundVGSContent.Object["spec"].(map[string]interface{}) + if spec["deletionPolicy"] != "Delete" { + spec["deletionPolicy"] = "Delete" + boundVGSContent, err = dc.Resource(utils.VolumeGroupSnapshotContentGVR).Update(ctx, boundVGSContent, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update VGSContent %q: %w", boundVGSContent.GetName(), err) + } + } + } + r.VGSContent = boundVGSContent + + // Delete the VolumeGroupSnapshot + if err := dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(vgsNamespace).Delete(ctx, vgsName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete VGS %q: %w", vgsName, err) + } + + // Wait for VolumeGroupSnapshot deleted + if err := utils.WaitForNamespacedGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotGVR, vgsNamespace, vgsName, framework.Poll, timeouts.SnapshotDelete); err != nil { + return fmt.Errorf("failed waiting for VGS %q deletion: %w", vgsName, err) + } + + // Wait for VolumeSnapshots owned by this group to be gone + if err := utils.WaitForOwnedResourcesDeleted(ctx, dc, utils.SnapshotGVR, vgsNamespace, vgsUID, framework.Poll, timeouts.SnapshotDelete); err != nil { + return fmt.Errorf("failed waiting for owned snapshots deletion of VGS %q: %w", vgsName, err) + } + + // Wait for all VolumeSnapshotsContents owned by this group to be gone + for _, contentName := range contentNamesSet.List() { + if err := utils.WaitForGVRDeletion(ctx, dc, utils.SnapshotContentGVR, contentName, framework.Poll, timeouts.SnapshotDelete); err != nil { + cleanupErrs = append(cleanupErrs, fmt.Errorf("failed waiting for VSC %q deletion: %w", contentName, err)) + } + } + if len(cleanupErrs) > 0 { + return utilerrors.NewAggregate(cleanupErrs) + } + + // Wait for VolumeGroupSnapshotContent deleted + if boundVGSContent != nil { + err = utils.WaitForGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotContentGVR, boundVGSContent.GetName(), framework.Poll, timeouts.SnapshotDelete) + if err == nil { + r.VGSContent = nil + } else { + return fmt.Errorf("failed waiting for VGSContent %q deletion: %w", vgsName, err) + } + } return nil } +// CleanupVGSClass deletes the VolumeGroupSnapshotClass. +func (r *VolumeGroupSnapshotResource) CleanupVGSClass(ctx context.Context, timeouts *framework.TimeoutContext) error { + if r.VGSClass == nil { + return nil + } + + dc := r.Config.Framework.DynamicClient + vgsClassName := r.VGSClass.GetName() + framework.Logf("deleting groupSnapshotClass %q", vgsClassName) + + err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, vgsClassName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete groupSnapshotClass %q: %w", vgsClassName, err) + } + + if err = utils.WaitForGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotClassGVR, vgsClassName, framework.Poll, timeouts.SnapshotDelete); err != nil { + return fmt.Errorf("failed waiting for groupSnapshotClass %q deletion: %w", vgsClassName, err) + } + + framework.Logf("successfully deleted groupSnapshotClass %q", vgsClassName) + return nil +} + +// CleanupResource deletes the VolumeGroupSnapshotClass and VolumeGroupSnapshot objects using a dynamic client, +// and ignores not found errors. +func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeouts *framework.TimeoutContext) error { + var cleanupErrs []error + + if err := r.CleanupVGS(ctx, timeouts); err != nil { + cleanupErrs = append(cleanupErrs, err) + } + + if err := r.CleanupVGSClass(ctx, timeouts); err != nil { + cleanupErrs = append(cleanupErrs, err) + } + + return utilerrors.NewAggregate(cleanupErrs) +} + // CreateVolumeGroupSnapshotResource creates a VolumeGroupSnapshotResource object with the given parameters. func CreateVolumeGroupSnapshotResource(ctx context.Context, sDriver VolumeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *VolumeGroupSnapshotResource { vgsClass, snapshot, vgsc := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters) diff --git a/test/e2e/storage/testsuites/volume_group_snapshottable.go b/test/e2e/storage/testsuites/volume_group_snapshottable.go index f603ad6f3c8..5f3dd08b2a4 100644 --- a/test/e2e/storage/testsuites/volume_group_snapshottable.go +++ b/test/e2e/storage/testsuites/volume_group_snapshottable.go @@ -28,6 +28,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" @@ -147,6 +148,10 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework. "app": statefulSetName, }, }, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -226,28 +231,57 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework. return originalMntTestData } - cleanup := func(ctx context.Context) { + cleanupResources := func(ctx context.Context) { if groupTest.statefulSet != nil { - framework.Logf("Deleting StatefulSet %s", groupTest.statefulSet.Name) - err := cs.AppsV1().StatefulSets(f.Namespace.Name).Delete(ctx, groupTest.statefulSet.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err, "failed to delete StatefulSet %s", groupTest.statefulSet.Name) + framework.Logf("deleting StatefulSet %s", groupTest.statefulSet.Name) + e2estatefulset.DeleteAllStatefulSets(ctx, cs, groupTest.statefulSet.Namespace) } - for _, volumeResource := range groupTest.volumeResources { - if volumeResource != nil { - framework.Logf("Deleting volume resource") - err := volumeResource.CleanupResource(ctx) - framework.ExpectNoError(err, "failed to delete volume resource") + + var cleanupVGSErrs []error + for _, vgsr := range groupTest.snapshots { + if vgsr == nil || vgsr.VGS == nil { + framework.Logf("Skipping cleanup: VolumeGroupSnapshotResource or VGS is nil") + continue + } + + vgsName := vgsr.VGS.GetName() + vgsNamespace := vgsr.VGS.GetNamespace() + + framework.Logf("deleting VolumeGroupSnapshotResource %s/%s", vgsNamespace, vgsName) + err := vgsr.CleanupResource(ctx, f.Timeouts) + if err != nil { + cleanupVGSErrs = append(cleanupVGSErrs, err) + framework.Logf("Warning: failed to delete VolumeGroupSnapshotResource %s/%s: %v", vgsNamespace, vgsName, err) + } else { + framework.Logf("deleted VolumeGroupSnapshotResource %s/%s", vgsNamespace, vgsName) } } + framework.ExpectNoError(utilerrors.NewAggregate(cleanupVGSErrs), "failed to delete VGS resources") + + var cleanupVolumeErrs []error + for _, volumeResource := range groupTest.volumeResources { + if volumeResource == nil || volumeResource.Pvc == nil { + continue + } + + ns, name := volumeResource.Pvc.Namespace, volumeResource.Pvc.Name + framework.Logf("deleting volume resource %s/%s", ns, name) + if err := volumeResource.CleanupResource(ctx); err != nil { + cleanupVolumeErrs = append(cleanupVolumeErrs, fmt.Errorf("%s/%s: %w", ns, name, err)) + framework.Logf("Warning: failed to delete volume resource %s/%s: %v", ns, name, err) + } else { + framework.Logf("deleted volume resource %s/%s", ns, name) + } + } + framework.ExpectNoError(utilerrors.NewAggregate(cleanupVolumeErrs), "failed to delete volume resources") } ginkgo.It("should create snapshots for StatefulSet volumes and verify data consistency after restore", func(ctx context.Context) { init(ctx) createStatefulSetAndVolumes(ctx) - ginkgo.DeferCleanup(cleanup) + ginkgo.DeferCleanup(cleanupResources) originalMntTestData := writeTestDataToVolumes(ctx) - snapshot := storageframework.CreateVolumeGroupSnapshotResource(ctx, snapshottableDriver, groupTest.config, pattern, labelValue, f.Namespace.Name, f.Timeouts, map[string]string{"deletionPolicy": pattern.SnapshotDeletionPolicy.String()}) groupTest.snapshots = append(groupTest.snapshots, snapshot) diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index 9a40ff43f24..281b8f979bf 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -37,6 +37,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -687,6 +688,52 @@ func WaitForNamespacedGVRDeletion(ctx context.Context, c dynamic.Interface, gvr return fmt.Errorf("%s %s in namespace %s is not deleted within %v", gvr.Resource, objectName, ns, timeout) } +// WaitForOwnedResourcesDeleted waits until all objects of the given GVR +// in the namespace (or cluster-scoped if ns=="") with the specified ownerUID are deleted. +func WaitForOwnedResourcesDeleted(ctx context.Context, + c dynamic.Interface, + gvr schema.GroupVersionResource, + ns string, // empty for cluster-scoped resources + ownerUID types.UID, + poll, timeout time.Duration) error { + scope := "cluster-scoped" + if ns != "" { + scope = fmt.Sprintf("namespace %s", ns) + } + framework.Logf("Waiting up to %v for all %s %s owned by %s to be deleted", timeout, gvr.Resource, scope, ownerUID) + + successful := WaitUntil(poll, timeout, func() bool { + var list *unstructured.UnstructuredList + var err error + if ns == "" { + list, err = c.Resource(gvr).List(ctx, metav1.ListOptions{}) + } else { + list, err = c.Resource(gvr).Namespace(ns).List(ctx, metav1.ListOptions{}) + } + if err != nil { + framework.Logf("List %s %s returned error: %v", gvr.Resource, scope, err) + return false + } + + for _, item := range list.Items { + for _, owner := range item.GetOwnerReferences() { + if owner.UID == ownerUID { + framework.Logf("%s %s is still present, waiting...", gvr.Resource, item.GetName()) + return false + } + } + } + + framework.Logf("All %s %s owned by %s have been deleted", gvr.Resource, scope, ownerUID) + return true + }) + + if successful { + return nil + } + return fmt.Errorf("some %s %s owned by %s were not deleted within %v", gvr.Resource, scope, ownerUID, timeout) +} + // WaitUntil runs checkDone until a timeout is reached func WaitUntil(poll, timeout time.Duration, checkDone func() bool) bool { // TODO (pohly): replace with gomega.Eventually