Merge pull request #93196 from AndiLi99/AndiLi99/preprovisoned-snapshot-test

Andi li99/preprovisoned snapshot test

test/e2e/storage/external/external.go

@@ -251,7 +251,7 @@ func (d *driverDefinition) SkipUnsupportedTest(pattern testpatterns.TestPattern)
 	switch pattern.SnapshotType {
 	case "":
 		supported = true
-	case testpatterns.DynamicCreatedSnapshot:
+	case testpatterns.DynamicCreatedSnapshot, testpatterns.PreprovisionedCreatedSnapshot:
 		if d.SnapshotClass.FromName || d.SnapshotClass.FromFile != "" || d.SnapshotClass.FromExistingClassName != "" {
 			supported = true
 		}

test/e2e/storage/testpatterns/testpattern.go

@@ -70,6 +70,10 @@ var (
 	RetainSnapshot TestSnapshotDeletionPolicy = "Retain"
 )
 
+func (t TestSnapshotDeletionPolicy) String() string {
+	return string(t)
+}
+
 // TestPattern represents a combination of parameters to be tested in a TestSuite
 type TestPattern struct {
 	Name                   string                      // Name of TestPattern

test/e2e/storage/testsuites/BUILD

@@ -35,9 +35,11 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",

test/e2e/storage/testsuites/base.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
@@ -571,6 +572,57 @@ func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.U
 
 	return snapshot
 }
+func getPreProvisionedSnapshot(snapName, ns, snapshotContentName string) *unstructured.Unstructured {
+	snapshot := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       "VolumeSnapshot",
+			"apiVersion": snapshotAPIVersion,
+			"metadata": map[string]interface{}{
+				"name":      snapName,
+				"namespace": ns,
+			},
+			"spec": map[string]interface{}{
+				"source": map[string]interface{}{
+					"volumeSnapshotContentName": snapshotContentName,
+				},
+			},
+		},
+	}
+
+	return snapshot
+}
+func getPreProvisionedSnapshotContent(snapcontentName, snapshotName, snapshotNamespace, snapshotHandle, deletionPolicy, csiDriverName string) *unstructured.Unstructured {
+	snapshotContent := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       "VolumeSnapshotContent",
+			"apiVersion": snapshotAPIVersion,
+			"metadata": map[string]interface{}{
+				"name": snapcontentName,
+			},
+			"spec": map[string]interface{}{
+				"source": map[string]interface{}{
+					"snapshotHandle": snapshotHandle,
+				},
+				"volumeSnapshotRef": map[string]interface{}{
+					"name":      snapshotName,
+					"namespace": snapshotNamespace,
+				},
+				"driver":         csiDriverName,
+				"deletionPolicy": deletionPolicy,
+			},
+		},
+	}
+
+	return snapshotContent
+}
+
+func getPreProvisionedSnapshotContentName(uuid types.UID) string {
+	return fmt.Sprintf("pre-provisioned-snapcontent-%s", string(uuid))
+}
+
+func getPreProvisionedSnapshotName(uuid types.UID) string {
+	return fmt.Sprintf("pre-provisioned-snapshot-%s", string(uuid))
+}
 
 // StartPodLogs begins capturing log output and events from current
 // and future pods running in the namespace of the framework. That

test/e2e/storage/testsuites/snapshottable.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -75,6 +76,8 @@ func InitSnapshottableTestSuite() TestSuite {
 			TestPatterns: []testpatterns.TestPattern{
 				testpatterns.DynamicSnapshotDelete,
 				testpatterns.DynamicSnapshotRetain,
+				testpatterns.PreprovisionedSnapshotDelete,
+				testpatterns.PreprovisionedSnapshotRetain,
 			},
 			SupportedSizeRange: e2evolume.SizeRange{
 				Min: "1Mi",
@@ -94,7 +97,6 @@ func (s *snapshottableTestSuite) SkipRedundantSuite(driver TestDriver, pattern t
 func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatterns.TestPattern) {
 	ginkgo.BeforeEach(func() {
 		// Check preconditions.
-		framework.ExpectEqual(pattern.SnapshotType, testpatterns.DynamicCreatedSnapshot)
 		dInfo := driver.GetDriverInfo()
 		ok := false
 		sDriver, ok = driver.(SnapshottableTestDriver)
@@ -137,14 +139,15 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt
 			config, driverCleanup = driver.PrepareTest(f)
 			cleanupSteps = append(cleanupSteps, driverCleanup)
 
-			volumeResource := CreateVolumeResource(dDriver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange)
+			var volumeResource *VolumeResource
+			cleanupSteps = append(cleanupSteps, func() {
+				framework.ExpectNoError(volumeResource.CleanupResource())
+			})
+			volumeResource = CreateVolumeResource(dDriver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange)
 
 			pvc = volumeResource.Pvc
 			sc = volumeResource.Sc
 			claimSize = pvc.Spec.Resources.Requests.Storage().String()
-			cleanupSteps = append(cleanupSteps, func() {
-				framework.ExpectNoError(volumeResource.CleanupResource())
-			})
 
 			ginkgo.By("starting a pod to use the claim")
 			originalMntTestData = fmt.Sprintf("hello from %s namespace", pvc.GetNamespace())
@@ -196,31 +199,16 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt
 			)
 
 			ginkgo.BeforeEach(func() {
-				sr := CreateSnapshotResource(sDriver, config, pattern, pvc.GetName(), pvc.GetNamespace())
-				vs = sr.Vs
-				vscontent = sr.Vscontent
-				vsc = sr.Vsclass
+				var sr *SnapshotResource
 				cleanupSteps = append(cleanupSteps, func() {
 					framework.ExpectNoError(sr.CleanupResource())
 				})
+				sr = CreateSnapshotResource(sDriver, config, pattern, pvc.GetName(), pvc.GetNamespace())
+				vs = sr.Vs
+				vscontent = sr.Vscontent
+				vsc = sr.Vsclass
 			})
-
-			ginkgo.It("should delete the VolumeSnapshotContent according to its deletion policy", func() {
-				err = DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout)
-				framework.ExpectNoError(err)
-
-				switch pattern.SnapshotDeletionPolicy {
-				case testpatterns.DeleteSnapshot:
-					ginkgo.By("checking the SnapshotContent has been deleted")
-					err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout)
-					framework.ExpectNoError(err)
-				case testpatterns.RetainSnapshot:
-					ginkgo.By("checking the SnapshotContent has not been deleted")
-					err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), 1*time.Second /* poll */, 30*time.Second /* timeout */)
-					framework.ExpectError(err)
-				}
-			})
-			ginkgo.It("should create snapshot objects correctly", func() {
+			ginkgo.It("should check snapshot fields, check restore correctly works after modifying source data, check deletion", func() {
 				ginkgo.By("checking the snapshot")
 				// Get new copy of the snapshot
 				vs, err = dc.Resource(SnapshotGVR).Namespace(vs.GetNamespace()).Get(context.TODO(), vs.GetName(), metav1.GetOptions{})
@@ -237,11 +225,14 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt
 
 				// Check SnapshotContent properties
 				ginkgo.By("checking the SnapshotContent")
-				framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], vsc.GetName())
+				// PreprovisionedCreatedSnapshot do not need to set volume snapshot class name
+				if pattern.SnapshotType != testpatterns.PreprovisionedCreatedSnapshot {
+					framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], vsc.GetName())
+				}
 				framework.ExpectEqual(volumeSnapshotRef["name"], vs.GetName())
 				framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace())
-			})
-			ginkgo.It("should restore from snapshot with saved data after modifying source data", func() {
+
+				ginkgo.By("Modifying source data test")
 				var restoredPVC *v1.PersistentVolumeClaim
 				var restoredPod *v1.Pod
 				modifiedMntTestData := fmt.Sprintf("modified data from %s namespace", pvc.GetNamespace())
@@ -258,14 +249,13 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt
 				}, config.Framework.Namespace.Name)
 
 				group := "snapshot.storage.k8s.io"
-				dataSourceRef := &v1.TypedLocalObjectReference{
+
+				restoredPVC.Spec.DataSource = &v1.TypedLocalObjectReference{
 					APIGroup: &group,
 					Kind:     "VolumeSnapshot",
 					Name:     vs.GetName(),
 				}
 
-				restoredPVC.Spec.DataSource = dataSourceRef
-
 				restoredPVC, err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Create(context.TODO(), restoredPVC, metav1.CreateOptions{})
 				framework.ExpectNoError(err)
 				cleanupSteps = append(cleanupSteps, func() {
@@ -280,15 +270,30 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt
 				ginkgo.By("starting a pod to use the claim")
 
 				restoredPod = StartInPodWithVolume(cs, restoredPVC.Namespace, restoredPVC.Name, "restored-pvc-tester", "sleep 300", config.ClientNodeSelection)
-				framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(cs, restoredPod.Name, restoredPod.Namespace))
 				cleanupSteps = append(cleanupSteps, func() {
 					StopPod(cs, restoredPod)
 				})
+				framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(cs, restoredPod.Name, restoredPod.Namespace))
 
 				command = "cat /mnt/test/data"
 				actualData, err := utils.PodExec(f, restoredPod, command)
 				framework.ExpectNoError(err)
 				framework.ExpectEqual(actualData, originalMntTestData)
+
+				ginkgo.By("should delete the VolumeSnapshotContent according to its deletion policy")
+				err = DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout)
+				framework.ExpectNoError(err)
+
+				switch pattern.SnapshotDeletionPolicy {
+				case testpatterns.DeleteSnapshot:
+					ginkgo.By("checking the SnapshotContent has been deleted")
+					err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout)
+					framework.ExpectNoError(err)
+				case testpatterns.RetainSnapshot:
+					ginkgo.By("checking the SnapshotContent has not been deleted")
+					err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), 1*time.Second /* poll */, 30*time.Second /* timeout */)
+					framework.ExpectError(err)
+				}
 			})
 		})
 	})
@@ -335,7 +340,7 @@ func DeleteAndWaitSnapshot(dc dynamic.Interface, ns string, snapshotName string,
 	}
 
 	ginkgo.By("checking the Snapshot has been deleted")
-	err = utils.WaitForGVRDeletion(dc, SnapshotGVR, snapshotName, poll, timeout)
+	err = utils.WaitForNamespacedGVRDeletion(dc, SnapshotGVR, ns, snapshotName, poll, timeout)
 
 	return err
 }
@@ -354,6 +359,10 @@ type SnapshotResource struct {
 // different test pattern snapshot provisioning and deletion policy
 func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConfig, pattern testpatterns.TestPattern, pvcName string, pvcNamespace string) *SnapshotResource {
 	var err error
+	if pattern.SnapshotType != testpatterns.DynamicCreatedSnapshot && pattern.SnapshotType != testpatterns.PreprovisionedCreatedSnapshot {
+		err = fmt.Errorf("SnapshotType must be set to either DynamicCreatedSnapshot or PreprovisionedCreatedSnapshot")
+		framework.ExpectNoError(err)
+	}
 	r := SnapshotResource{
 		Config:  config,
 		Pattern: pattern,
@@ -365,50 +374,95 @@ func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConf
 	if r.Vsclass == nil {
 		framework.Failf("Failed to get snapshot class based on test config")
 	}
-	r.Vsclass.Object["deletionPolicy"] = pattern.SnapshotDeletionPolicy
+	r.Vsclass.Object["deletionPolicy"] = pattern.SnapshotDeletionPolicy.String()
 
 	r.Vsclass, err = dc.Resource(SnapshotClassGVR).Create(context.TODO(), r.Vsclass, metav1.CreateOptions{})
 	framework.ExpectNoError(err)
 
-	r.Vsclass, err = dc.Resource(SnapshotClassGVR).Namespace(r.Vsclass.GetNamespace()).Get(context.TODO(), r.Vsclass.GetName(), metav1.GetOptions{})
+	r.Vsclass, err = dc.Resource(SnapshotClassGVR).Get(context.TODO(), r.Vsclass.GetName(), metav1.GetOptions{})
 	framework.ExpectNoError(err)
 
-	switch pattern.SnapshotType {
-	case testpatterns.DynamicCreatedSnapshot:
-		ginkgo.By("creating a VolumeSnapshot")
-		// prepare a dynamically provisioned volume snapshot with certain data
-		r.Vs = getSnapshot(pvcName, pvcNamespace, r.Vsclass.GetName())
+	ginkgo.By("creating a dynamic VolumeSnapshot")
+	// prepare a dynamically provisioned volume snapshot with certain data
+	r.Vs = getSnapshot(pvcName, pvcNamespace, r.Vsclass.GetName())
 
+	r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{})
+	framework.ExpectNoError(err)
+
+	err = WaitForSnapshotReady(dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
+	framework.ExpectNoError(err)
+
+	r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(context.TODO(), r.Vs.GetName(), metav1.GetOptions{})
+
+	snapshotStatus := r.Vs.Object["status"].(map[string]interface{})
+	snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string)
+	framework.Logf("received snapshotStatus %v", snapshotStatus)
+	framework.Logf("snapshotContentName %s", snapshotContentName)
+	framework.ExpectNoError(err)
+
+	r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{})
+	framework.ExpectNoError(err)
+
+	if pattern.SnapshotType == testpatterns.PreprovisionedCreatedSnapshot {
+		// prepare a pre-provisioned VolumeSnapshotContent with certain data
+		// Because this could be run with an external CSI driver, we have no way
+		// to pre-provision the snapshot as we normally would using their API.
+		// We instead dynamically take a snapshot (above step), delete the old snapshot,
+		// and create another snapshot using the first snapshot's snapshot handle.
+
+		ginkgo.By("updating the snapshot content deletion policy to retain")
+		r.Vscontent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Retain"
+
+		r.Vscontent, err = dc.Resource(SnapshotContentGVR).Update(context.TODO(), r.Vscontent, metav1.UpdateOptions{})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("recording the volume handle and snapshotHandle")
+		snapshotHandle := r.Vscontent.Object["status"].(map[string]interface{})["snapshotHandle"].(string)
+		framework.Logf("Recording snapshot handle: %s", snapshotHandle)
+		csiDriverName := r.Vsclass.Object["driver"].(string)
+
+		// If the deletion policy is retain on vscontent:
+		// when vs is deleted vscontent will not be deleted
+		// when the vscontent is manually deleted then the underlying snapshot resource will not be deleted.
+		// We exploit this to create a snapshot resource from which we can create a preprovisioned snapshot
+		ginkgo.By("deleting the snapshot and snapshot content")
+		err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Delete(context.TODO(), r.Vs.GetName(), metav1.DeleteOptions{})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("checking the Snapshot has been deleted")
+		err = utils.WaitForNamespacedGVRDeletion(dc, SnapshotGVR, r.Vs.GetName(), r.Vs.GetNamespace(), framework.Poll, framework.SnapshotDeleteTimeout)
+		framework.ExpectNoError(err)
+
+		err = dc.Resource(SnapshotContentGVR).Delete(context.TODO(), r.Vscontent.GetName(), metav1.DeleteOptions{})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("checking the Snapshot content has been deleted")
+		err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, r.Vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("creating a snapshot content with the snapshot handle")
+		uuid := uuid.NewUUID()
+
+		snapName := getPreProvisionedSnapshotName(uuid)
+		snapcontentName := getPreProvisionedSnapshotContentName(uuid)
+
+		r.Vscontent = getPreProvisionedSnapshotContent(snapcontentName, snapName, pvcNamespace, snapshotHandle, pattern.SnapshotDeletionPolicy.String(), csiDriverName)
+		r.Vscontent, err = dc.Resource(SnapshotContentGVR).Create(context.TODO(), r.Vscontent, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("creating a snapshot with that snapshot content")
+		r.Vs = getPreProvisionedSnapshot(snapName, pvcNamespace, snapcontentName)
 		r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{})
 		framework.ExpectNoError(err)
 
 		err = WaitForSnapshotReady(dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
 		framework.ExpectNoError(err)
 
+		ginkgo.By("getting the snapshot and snapshot content")
 		r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(context.TODO(), r.Vs.GetName(), metav1.GetOptions{})
-
-		snapshotStatus := r.Vs.Object["status"].(map[string]interface{})
-		snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string)
-		framework.Logf("received snapshotStatus %v", snapshotStatus)
-		framework.Logf("snapshotContentName %s", snapshotContentName)
 		framework.ExpectNoError(err)
 
-		r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{})
-		framework.ExpectNoError(err)
-	case testpatterns.PreprovisionedCreatedSnapshot:
-		// prepare a pre-provisioned VolumeSnapshotContent with certain data
-		// Because this could be run with an external CSI driver, we have no way
-		// to pre-provision the snapshot as we normally would using their API.
-		// We instead dynamically take a snapshot and create another snapshot using
-		// the first snapshot's snapshot handle.
-		ginkgo.Skip("Preprovisioned test not implemented")
-		ginkgo.By("taking a snapshot with deletion policy retain")
-		ginkgo.By("recording the volume handle and status.snapshotHandle")
-		ginkgo.By("deleting the snapshot and snapshot content") // TODO: test what happens when I have two snapshot content that refer to the same content
-		ginkgo.By("creating a snapshot content with the snapshot handle")
-		ginkgo.By("creating a snapshot with that snapshot content")
-	default:
-		err = fmt.Errorf("SnapshotType must be set to either DynamicCreatedSnapshot or PreprovisionedCreatedSnapshot")
+		r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), r.Vscontent.GetName(), metav1.GetOptions{})
 		framework.ExpectNoError(err)
 	}
 	return &r
@@ -451,7 +505,7 @@ func (sr *SnapshotResource) CleanupResource() error {
 				// the volume snapshot is not bound to snapshot content yet
 				err = dc.Resource(SnapshotGVR).Namespace(sr.Vs.GetNamespace()).Delete(context.TODO(), sr.Vs.GetName(), metav1.DeleteOptions{})
 				framework.ExpectNoError(err)
-				err = utils.WaitForGVRDeletion(dc, SnapshotGVR, sr.Vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout)
+				err = utils.WaitForNamespacedGVRDeletion(dc, SnapshotGVR, sr.Vs.GetName(), sr.Vs.GetNamespace(), framework.Poll, framework.SnapshotDeleteTimeout)
 				framework.ExpectNoError(err)
 			default:
 				cleanupErrs = append(cleanupErrs, err)
@@ -463,7 +517,7 @@ func (sr *SnapshotResource) CleanupResource() error {
 		}
 	}
 	if sr.Vscontent != nil {
-		framework.Logf("deleting snapshot content %q/%q", sr.Vscontent.GetNamespace(), sr.Vscontent.GetName())
+		framework.Logf("deleting snapshot content %q", sr.Vscontent.GetName())
 
 		sr.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), sr.Vscontent.GetName(), metav1.GetOptions{})
 		switch {
@@ -476,7 +530,7 @@ func (sr *SnapshotResource) CleanupResource() error {
 				sr.Vscontent, err = dc.Resource(SnapshotContentGVR).Update(context.TODO(), sr.Vscontent, metav1.UpdateOptions{})
 				framework.ExpectNoError(err)
 			}
-			err = dc.Resource(SnapshotContentGVR).Namespace(sr.Vscontent.GetNamespace()).Delete(context.TODO(), sr.Vscontent.GetName(), metav1.DeleteOptions{})
+			err = dc.Resource(SnapshotContentGVR).Delete(context.TODO(), sr.Vscontent.GetName(), metav1.DeleteOptions{})
 			framework.ExpectNoError(err)
 
 			err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, sr.Vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout)
@@ -488,9 +542,9 @@ func (sr *SnapshotResource) CleanupResource() error {
 		}
 	}
 	if sr.Vsclass != nil {
-		framework.Logf("deleting snapshot class %q/%q", sr.Vsclass.GetNamespace(), sr.Vsclass.GetName())
+		framework.Logf("deleting snapshot class %q", sr.Vsclass.GetName())
 		// typically this snapshot class has already been deleted
-		err = dc.Resource(SnapshotClassGVR).Namespace(sr.Vsclass.GetNamespace()).Delete(context.TODO(), sr.Vsclass.GetName(), metav1.DeleteOptions{})
+		err = dc.Resource(SnapshotClassGVR).Delete(context.TODO(), sr.Vsclass.GetName(), metav1.DeleteOptions{})
 		if err != nil && !apierrors.IsNotFound(err) {
 			framework.Failf("Error deleting snapshot class %q. Error: %v", sr.Vsclass.GetName(), err)
 		}

test/e2e/storage/utils/utils.go

@@ -727,7 +727,7 @@ func CreateDriverNamespace(f *framework.Framework) *v1.Namespace {
 	return namespace
 }
 
-// WaitForGVRDeletion waits until an object has been deleted
+// WaitForGVRDeletion waits until a non-namespaced object has been deleted
 func WaitForGVRDeletion(c dynamic.Interface, gvr schema.GroupVersionResource, objectName string, poll, timeout time.Duration) error {
 	framework.Logf("Waiting up to %v for %s %s to be deleted", timeout, gvr.Resource, objectName)
 
@@ -737,7 +737,7 @@ func WaitForGVRDeletion(c dynamic.Interface, gvr schema.GroupVersionResource, ob
 			framework.Logf("%s %v is not found and has been deleted", gvr.Resource, objectName)
 			return true
 		} else if err != nil {
-			framework.Logf("Get $s %v returned an error: %v", objectName, err.Error())
+			framework.Logf("Get %s returned an error: %v", objectName, err.Error())
 		} else {
 			framework.Logf("%s %v has been found and is not deleted", gvr.Resource, objectName)
 		}
@@ -750,6 +750,29 @@ func WaitForGVRDeletion(c dynamic.Interface, gvr schema.GroupVersionResource, ob
 	return fmt.Errorf("%s %s is not deleted within %v", gvr.Resource, objectName, timeout)
 }
 
+// WaitForNamespacedGVRDeletion waits until a namespaced object has been deleted
+func WaitForNamespacedGVRDeletion(c dynamic.Interface, gvr schema.GroupVersionResource, ns, objectName string, poll, timeout time.Duration) error {
+	framework.Logf("Waiting up to %v for %s %s to be deleted", timeout, gvr.Resource, objectName)
+
+	if successful := WaitUntil(poll, timeout, func() bool {
+		_, err := c.Resource(gvr).Namespace(ns).Get(context.TODO(), objectName, metav1.GetOptions{})
+		if err != nil && apierrors.IsNotFound(err) {
+			framework.Logf("%s %s is not found in namespace %s and has been deleted", gvr.Resource, objectName, ns)
+			return true
+		} else if err != nil {
+			framework.Logf("Get %s in namespace %s returned an error: %v", objectName, ns, err.Error())
+		} else {
+			framework.Logf("%s %s has been found in namespace %s and is not deleted", gvr.Resource, objectName, ns)
+		}
+
+		return false
+	}); successful {
+		return nil
+	}
+
+	return fmt.Errorf("%s %s in namespace %s is not deleted within %v", gvr.Resource, objectName, ns, timeout)
+}
+
 // WaitUntil runs checkDone until a timeout is reached
 func WaitUntil(poll, timeout time.Duration, checkDone func() bool) bool {
 	for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) {