mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-18 04:54:54 +00:00
test: refine vgs resources clean up (#135250)
* test: refine vgs resources clean up Signed-off-by: Penghao <pewang@redhat.com> * fix: refine structure Signed-off-by: Penghao <pewang@redhat.com> * fix: typo and proper data structure usage Signed-off-by: Penghao <pewang@redhat.com> --------- Signed-off-by: Penghao <pewang@redhat.com>
This commit is contained in:
Penghao
GitHub
@@ -22,8 +22,11 @@ import (
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/gomega"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
"k8s.io/kubernetes/test/e2e/storage/utils"
|
||||
)
|
||||
@@ -110,15 +113,149 @@ func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VolumeGroupSnapshott
|
||||
return gsclass, volumeGroupSnapshot, vgsc
|
||||
}
|
||||
|
||||
// CleanupResource deletes the VolumeGroupSnapshotClass and VolumeGroupSnapshot objects using a dynamic client.
|
||||
func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeouts *framework.TimeoutContext) error {
|
||||
defer ginkgo.GinkgoRecover()
|
||||
// CleanupVGS deletes the VolumeGroupSnapshot and ensures the bound VolumeGroupSnapshotContent has Delete policy.
|
||||
// It waits for the VolumeGroupSnapshot, its owned VolumeSnapshots, VolumeSnapshotContents, and the bound
|
||||
// VolumeGroupSnapshotContent to be fully deleted to prevent resource leaks.
|
||||
func (r *VolumeGroupSnapshotResource) CleanupVGS(ctx context.Context, timeouts *framework.TimeoutContext) error {
|
||||
if r.VGS == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var cleanupErrs []error
|
||||
dc := r.Config.Framework.DynamicClient
|
||||
err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, r.VGSClass.GetName(), metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err, "Failed to delete volume group snapshot class")
|
||||
vgsNamespace := r.VGS.GetNamespace()
|
||||
vgsName := r.VGS.GetName()
|
||||
vgsUID := r.VGS.GetUID()
|
||||
framework.Logf("deleting groupSnapshot %q/%q and ensuring content has Delete policy", vgsNamespace, vgsName)
|
||||
|
||||
vgs, err := dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(vgsNamespace).Get(ctx, vgsName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
// Hope that the underlying snapshot contents and resources are gone already
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to get VGS %q: %w", vgsName, err)
|
||||
}
|
||||
r.VGS = vgs
|
||||
|
||||
// Filter snapshots owned by this VGS
|
||||
vss, err := dc.Resource(utils.SnapshotGVR).Namespace(vgsNamespace).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
contentNamesSet := sets.NewString()
|
||||
for _, vs := range vss.Items {
|
||||
for _, owner := range vs.GetOwnerReferences() {
|
||||
if owner.Kind == "VolumeGroupSnapshot" && owner.UID == vgsUID {
|
||||
if status, ok := vs.Object["status"].(map[string]interface{}); ok {
|
||||
if cName, ok := status["boundVolumeSnapshotContentName"].(string); ok && cName != "" {
|
||||
contentNamesSet.Insert(cName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
groupSnapshotStatus := r.VGS.Object["status"].(map[string]interface{})
|
||||
groupSnapshotContentName := groupSnapshotStatus["boundVolumeGroupSnapshotContentName"].(string)
|
||||
framework.Logf("received groupSnapshotStatus %v", groupSnapshotStatus)
|
||||
framework.Logf("groupSnapshotContentName %q", groupSnapshotContentName)
|
||||
|
||||
boundVGSContent, err := dc.Resource(utils.VolumeGroupSnapshotContentGVR).Get(ctx, groupSnapshotContentName, metav1.GetOptions{})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to get bound VGSContent %q: %w", boundVGSContent.GetName(), err)
|
||||
}
|
||||
|
||||
// Ensure deletion policy is set to Delete to prevent leaks
|
||||
if boundVGSContent != nil {
|
||||
spec := boundVGSContent.Object["spec"].(map[string]interface{})
|
||||
if spec["deletionPolicy"] != "Delete" {
|
||||
spec["deletionPolicy"] = "Delete"
|
||||
boundVGSContent, err = dc.Resource(utils.VolumeGroupSnapshotContentGVR).Update(ctx, boundVGSContent, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update VGSContent %q: %w", boundVGSContent.GetName(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
r.VGSContent = boundVGSContent
|
||||
|
||||
// Delete the VolumeGroupSnapshot
|
||||
if err := dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(vgsNamespace).Delete(ctx, vgsName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to delete VGS %q: %w", vgsName, err)
|
||||
}
|
||||
|
||||
// Wait for VolumeGroupSnapshot deleted
|
||||
if err := utils.WaitForNamespacedGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotGVR, vgsNamespace, vgsName, framework.Poll, timeouts.SnapshotDelete); err != nil {
|
||||
return fmt.Errorf("failed waiting for VGS %q deletion: %w", vgsName, err)
|
||||
}
|
||||
|
||||
// Wait for VolumeSnapshots owned by this group to be gone
|
||||
if err := utils.WaitForOwnedResourcesDeleted(ctx, dc, utils.SnapshotGVR, vgsNamespace, vgsUID, framework.Poll, timeouts.SnapshotDelete); err != nil {
|
||||
return fmt.Errorf("failed waiting for owned snapshots deletion of VGS %q: %w", vgsName, err)
|
||||
}
|
||||
|
||||
// Wait for all VolumeSnapshotsContents owned by this group to be gone
|
||||
for _, contentName := range contentNamesSet.List() {
|
||||
if err := utils.WaitForGVRDeletion(ctx, dc, utils.SnapshotContentGVR, contentName, framework.Poll, timeouts.SnapshotDelete); err != nil {
|
||||
cleanupErrs = append(cleanupErrs, fmt.Errorf("failed waiting for VSC %q deletion: %w", contentName, err))
|
||||
}
|
||||
}
|
||||
if len(cleanupErrs) > 0 {
|
||||
return utilerrors.NewAggregate(cleanupErrs)
|
||||
}
|
||||
|
||||
// Wait for VolumeGroupSnapshotContent deleted
|
||||
if boundVGSContent != nil {
|
||||
err = utils.WaitForGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotContentGVR, boundVGSContent.GetName(), framework.Poll, timeouts.SnapshotDelete)
|
||||
if err == nil {
|
||||
r.VGSContent = nil
|
||||
} else {
|
||||
return fmt.Errorf("failed waiting for VGSContent %q deletion: %w", vgsName, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupVGSClass deletes the VolumeGroupSnapshotClass.
|
||||
func (r *VolumeGroupSnapshotResource) CleanupVGSClass(ctx context.Context, timeouts *framework.TimeoutContext) error {
|
||||
if r.VGSClass == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
dc := r.Config.Framework.DynamicClient
|
||||
vgsClassName := r.VGSClass.GetName()
|
||||
framework.Logf("deleting groupSnapshotClass %q", vgsClassName)
|
||||
|
||||
err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, vgsClassName, metav1.DeleteOptions{})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to delete groupSnapshotClass %q: %w", vgsClassName, err)
|
||||
}
|
||||
|
||||
if err = utils.WaitForGVRDeletion(ctx, dc, utils.VolumeGroupSnapshotClassGVR, vgsClassName, framework.Poll, timeouts.SnapshotDelete); err != nil {
|
||||
return fmt.Errorf("failed waiting for groupSnapshotClass %q deletion: %w", vgsClassName, err)
|
||||
}
|
||||
|
||||
framework.Logf("successfully deleted groupSnapshotClass %q", vgsClassName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupResource deletes the VolumeGroupSnapshotClass and VolumeGroupSnapshot objects using a dynamic client,
|
||||
// and ignores not found errors.
|
||||
func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeouts *framework.TimeoutContext) error {
|
||||
var cleanupErrs []error
|
||||
|
||||
if err := r.CleanupVGS(ctx, timeouts); err != nil {
|
||||
cleanupErrs = append(cleanupErrs, err)
|
||||
}
|
||||
|
||||
if err := r.CleanupVGSClass(ctx, timeouts); err != nil {
|
||||
cleanupErrs = append(cleanupErrs, err)
|
||||
}
|
||||
|
||||
return utilerrors.NewAggregate(cleanupErrs)
|
||||
}
|
||||
|
||||
// CreateVolumeGroupSnapshotResource creates a VolumeGroupSnapshotResource object with the given parameters.
|
||||
func CreateVolumeGroupSnapshotResource(ctx context.Context, sDriver VolumeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *VolumeGroupSnapshotResource {
|
||||
vgsClass, snapshot, vgsc := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/test/e2e/feature"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
@@ -147,6 +148,10 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
|
||||
"app": statefulSetName,
|
||||
},
|
||||
},
|
||||
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
|
||||
WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
|
||||
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
|
||||
},
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
@@ -226,28 +231,57 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
|
||||
return originalMntTestData
|
||||
}
|
||||
|
||||
cleanup := func(ctx context.Context) {
|
||||
cleanupResources := func(ctx context.Context) {
|
||||
if groupTest.statefulSet != nil {
|
||||
framework.Logf("Deleting StatefulSet %s", groupTest.statefulSet.Name)
|
||||
err := cs.AppsV1().StatefulSets(f.Namespace.Name).Delete(ctx, groupTest.statefulSet.Name, metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err, "failed to delete StatefulSet %s", groupTest.statefulSet.Name)
|
||||
framework.Logf("deleting StatefulSet %s", groupTest.statefulSet.Name)
|
||||
e2estatefulset.DeleteAllStatefulSets(ctx, cs, groupTest.statefulSet.Namespace)
|
||||
}
|
||||
for _, volumeResource := range groupTest.volumeResources {
|
||||
if volumeResource != nil {
|
||||
framework.Logf("Deleting volume resource")
|
||||
err := volumeResource.CleanupResource(ctx)
|
||||
framework.ExpectNoError(err, "failed to delete volume resource")
|
||||
|
||||
var cleanupVGSErrs []error
|
||||
for _, vgsr := range groupTest.snapshots {
|
||||
if vgsr == nil || vgsr.VGS == nil {
|
||||
framework.Logf("Skipping cleanup: VolumeGroupSnapshotResource or VGS is nil")
|
||||
continue
|
||||
}
|
||||
|
||||
vgsName := vgsr.VGS.GetName()
|
||||
vgsNamespace := vgsr.VGS.GetNamespace()
|
||||
|
||||
framework.Logf("deleting VolumeGroupSnapshotResource %s/%s", vgsNamespace, vgsName)
|
||||
err := vgsr.CleanupResource(ctx, f.Timeouts)
|
||||
if err != nil {
|
||||
cleanupVGSErrs = append(cleanupVGSErrs, err)
|
||||
framework.Logf("Warning: failed to delete VolumeGroupSnapshotResource %s/%s: %v", vgsNamespace, vgsName, err)
|
||||
} else {
|
||||
framework.Logf("deleted VolumeGroupSnapshotResource %s/%s", vgsNamespace, vgsName)
|
||||
}
|
||||
}
|
||||
framework.ExpectNoError(utilerrors.NewAggregate(cleanupVGSErrs), "failed to delete VGS resources")
|
||||
|
||||
var cleanupVolumeErrs []error
|
||||
for _, volumeResource := range groupTest.volumeResources {
|
||||
if volumeResource == nil || volumeResource.Pvc == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
ns, name := volumeResource.Pvc.Namespace, volumeResource.Pvc.Name
|
||||
framework.Logf("deleting volume resource %s/%s", ns, name)
|
||||
if err := volumeResource.CleanupResource(ctx); err != nil {
|
||||
cleanupVolumeErrs = append(cleanupVolumeErrs, fmt.Errorf("%s/%s: %w", ns, name, err))
|
||||
framework.Logf("Warning: failed to delete volume resource %s/%s: %v", ns, name, err)
|
||||
} else {
|
||||
framework.Logf("deleted volume resource %s/%s", ns, name)
|
||||
}
|
||||
}
|
||||
framework.ExpectNoError(utilerrors.NewAggregate(cleanupVolumeErrs), "failed to delete volume resources")
|
||||
}
|
||||
|
||||
ginkgo.It("should create snapshots for StatefulSet volumes and verify data consistency after restore", func(ctx context.Context) {
|
||||
init(ctx)
|
||||
createStatefulSetAndVolumes(ctx)
|
||||
ginkgo.DeferCleanup(cleanup)
|
||||
ginkgo.DeferCleanup(cleanupResources)
|
||||
|
||||
originalMntTestData := writeTestDataToVolumes(ctx)
|
||||
|
||||
snapshot := storageframework.CreateVolumeGroupSnapshotResource(ctx, snapshottableDriver, groupTest.config, pattern, labelValue, f.Namespace.Name, f.Timeouts, map[string]string{"deletionPolicy": pattern.SnapshotDeletionPolicy.String()})
|
||||
groupTest.snapshots = append(groupTest.snapshots, snapshot)
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/client-go/dynamic"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
@@ -687,6 +688,52 @@ func WaitForNamespacedGVRDeletion(ctx context.Context, c dynamic.Interface, gvr
|
||||
return fmt.Errorf("%s %s in namespace %s is not deleted within %v", gvr.Resource, objectName, ns, timeout)
|
||||
}
|
||||
|
||||
// WaitForOwnedResourcesDeleted waits until all objects of the given GVR
|
||||
// in the namespace (or cluster-scoped if ns=="") with the specified ownerUID are deleted.
|
||||
func WaitForOwnedResourcesDeleted(ctx context.Context,
|
||||
c dynamic.Interface,
|
||||
gvr schema.GroupVersionResource,
|
||||
ns string, // empty for cluster-scoped resources
|
||||
ownerUID types.UID,
|
||||
poll, timeout time.Duration) error {
|
||||
scope := "cluster-scoped"
|
||||
if ns != "" {
|
||||
scope = fmt.Sprintf("namespace %s", ns)
|
||||
}
|
||||
framework.Logf("Waiting up to %v for all %s %s owned by %s to be deleted", timeout, gvr.Resource, scope, ownerUID)
|
||||
|
||||
successful := WaitUntil(poll, timeout, func() bool {
|
||||
var list *unstructured.UnstructuredList
|
||||
var err error
|
||||
if ns == "" {
|
||||
list, err = c.Resource(gvr).List(ctx, metav1.ListOptions{})
|
||||
} else {
|
||||
list, err = c.Resource(gvr).Namespace(ns).List(ctx, metav1.ListOptions{})
|
||||
}
|
||||
if err != nil {
|
||||
framework.Logf("List %s %s returned error: %v", gvr.Resource, scope, err)
|
||||
return false
|
||||
}
|
||||
|
||||
for _, item := range list.Items {
|
||||
for _, owner := range item.GetOwnerReferences() {
|
||||
if owner.UID == ownerUID {
|
||||
framework.Logf("%s %s is still present, waiting...", gvr.Resource, item.GetName())
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
framework.Logf("All %s %s owned by %s have been deleted", gvr.Resource, scope, ownerUID)
|
||||
return true
|
||||
})
|
||||
|
||||
if successful {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("some %s %s owned by %s were not deleted within %v", gvr.Resource, scope, ownerUID, timeout)
|
||||
}
|
||||
|
||||
// WaitUntil runs checkDone until a timeout is reached
|
||||
func WaitUntil(poll, timeout time.Duration, checkDone func() bool) bool {
|
||||
// TODO (pohly): replace with gomega.Eventually
|
||||
|
||||
Reference in New Issue
Block a user