From 9b0dff7686228e2b6f514cdd496e2ae6ab236b9b Mon Sep 17 00:00:00 2001 From: Andi Li Date: Wed, 8 Jul 2020 16:27:43 +0000 Subject: [PATCH 1/3] Refactor snapshottable and provisioning tests 1. Use ginkgo before each to do common setup 2. Use volume resource to create SC, PV, PVC and handle cleanup 3. Add SnapshotResource to handle creating and cleanup of VS, VSC, VSClass 4. Add test pattern for deletion policy: Delete vs Retain 5. Use test pattern to determine test behaviour 6. Add test pattern for preprovisioned snapshot (not implemented) These changes are made to consolidate common setup steps and stop resource leaks by waiting for objects to be deleted. --- test/e2e/storage/testpatterns/testpattern.go | 74 ++- test/e2e/storage/testsuites/provisioning.go | 45 +- test/e2e/storage/testsuites/snapshottable.go | 547 ++++++++++--------- 3 files changed, 361 insertions(+), 305 deletions(-) diff --git a/test/e2e/storage/testpatterns/testpattern.go b/test/e2e/storage/testpatterns/testpattern.go index f76ca315218..b1930daaa32 100644 --- a/test/e2e/storage/testpatterns/testpattern.go +++ b/test/e2e/storage/testpatterns/testpattern.go @@ -54,18 +54,31 @@ type TestSnapshotType string var ( // DynamicCreatedSnapshot represents a snapshot type for dynamic created snapshot DynamicCreatedSnapshot TestSnapshotType = "DynamicSnapshot" + // PreprovisionedCreatedSnapshot represents a snapshot type for pre-provisioned snapshot + PreprovisionedCreatedSnapshot TestSnapshotType = "PreprovisionedSnapshot" +) + +// TestSnapshotDeletionPolicy represents the deletion policy of the snapshot class +type TestSnapshotDeletionPolicy string + +var ( + // DeleteSnapshot represents delete policy + DeleteSnapshot TestSnapshotDeletionPolicy = "Delete" + // RetainSnapshot represents retain policy + RetainSnapshot TestSnapshotDeletionPolicy = "Retain" ) // TestPattern represents a combination of parameters to be tested in a TestSuite type TestPattern struct { - Name string // Name of TestPattern - FeatureTag string // featureTag for the TestSuite - VolType TestVolType // Volume type of the volume - FsType string // Fstype of the volume - VolMode v1.PersistentVolumeMode // PersistentVolumeMode of the volume - SnapshotType TestSnapshotType // Snapshot type of the snapshot - BindingMode storagev1.VolumeBindingMode // VolumeBindingMode of the volume - AllowExpansion bool // AllowVolumeExpansion flag of the StorageClass + Name string // Name of TestPattern + FeatureTag string // featureTag for the TestSuite + VolType TestVolType // Volume type of the volume + FsType string // Fstype of the volume + VolMode v1.PersistentVolumeMode // PersistentVolumeMode of the volume + SnapshotType TestSnapshotType // Snapshot type of the snapshot + SnapshotDeletionPolicy TestSnapshotDeletionPolicy // Deletion policy of the snapshot class + BindingMode storagev1.VolumeBindingMode // VolumeBindingMode of the volume + AllowExpansion bool // AllowVolumeExpansion flag of the StorageClass } var ( @@ -88,8 +101,10 @@ var ( } // DefaultFsDynamicPV is TestPattern for "Dynamic PV (default fs)" DefaultFsDynamicPV = TestPattern{ - Name: "Dynamic PV (default fs)", - VolType: DynamicPV, + Name: "Dynamic PV (default fs)", + VolType: DynamicPV, + SnapshotType: DynamicCreatedSnapshot, + SnapshotDeletionPolicy: DeleteSnapshot, } // Definitions for ext3 @@ -233,17 +248,42 @@ var ( } // BlockVolModeDynamicPV is TestPattern for "Dynamic PV (block)" BlockVolModeDynamicPV = TestPattern{ - Name: "Dynamic PV (block volmode)", - VolType: DynamicPV, - VolMode: v1.PersistentVolumeBlock, + Name: "Dynamic PV (block volmode)", + VolType: DynamicPV, + VolMode: v1.PersistentVolumeBlock, + SnapshotType: DynamicCreatedSnapshot, + SnapshotDeletionPolicy: DeleteSnapshot, } // Definitions for snapshot case - // DynamicSnapshot is TestPattern for "Dynamic snapshot" - DynamicSnapshot = TestPattern{ - Name: "Dynamic Snapshot", - SnapshotType: DynamicCreatedSnapshot, + // DynamicSnapshotDelete is TestPattern for "Dynamic snapshot" + DynamicSnapshotDelete = TestPattern{ + Name: "Dynamic Snapshot (delete policy)", + SnapshotType: DynamicCreatedSnapshot, + SnapshotDeletionPolicy: DeleteSnapshot, + VolType: DynamicPV, + } + // PreprovisionedSnapshotDelete is TestPattern for "Pre-provisioned snapshot" + PreprovisionedSnapshotDelete = TestPattern{ + Name: "Pre-provisioned Snapshot (delete policy)", + SnapshotType: PreprovisionedCreatedSnapshot, + SnapshotDeletionPolicy: DeleteSnapshot, + VolType: DynamicPV, + } + // DynamicSnapshotRetain is TestPattern for "Dynamic snapshot" + DynamicSnapshotRetain = TestPattern{ + Name: "Dynamic Snapshot (retain policy)", + SnapshotType: DynamicCreatedSnapshot, + SnapshotDeletionPolicy: RetainSnapshot, + VolType: DynamicPV, + } + // PreprovisionedSnapshotRetain is TestPattern for "Pre-provisioned snapshot" + PreprovisionedSnapshotRetain = TestPattern{ + Name: "Pre-provisioned Snapshot (retain policy)", + SnapshotType: PreprovisionedCreatedSnapshot, + SnapshotDeletionPolicy: RetainSnapshot, + VolType: DynamicPV, } // Definitions for volume expansion case diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 63070544b7a..6bbcf04ecae 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -30,7 +30,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -201,6 +200,9 @@ func (p *provisioningTestSuite) DefineTests(driver TestDriver, pattern testpatte if !dInfo.Capabilities[CapSnapshotDataSource] { e2eskipper.Skipf("Driver %q does not support populate data from snapshot - skipping", dInfo.Name) } + if !dInfo.SupportedFsType.Has(pattern.FsType) { + e2eskipper.Skipf("Driver %q does not support %q fs type - skipping", dInfo.Name, pattern.FsType) + } sDriver, ok := driver.(SnapshottableTestDriver) if !ok { @@ -211,10 +213,9 @@ func (p *provisioningTestSuite) DefineTests(driver TestDriver, pattern testpatte defer cleanup() dc := l.config.Framework.DynamicClient - vsc := sDriver.GetSnapshotClass(l.config) testConfig := convertTestConfig(l.config) expectedContent := fmt.Sprintf("Hello from namespace %s", f.Namespace.Name) - dataSource, cleanupFunc := prepareSnapshotDataSourceForProvisioning(f, testConfig, l.cs, dc, l.pvc, l.sc, vsc, pattern.VolMode, expectedContent) + dataSource, cleanupFunc := prepareSnapshotDataSourceForProvisioning(f, testConfig, l.config, pattern, l.cs, dc, l.pvc, l.sc, sDriver, pattern.VolMode, expectedContent) defer cleanupFunc() l.pvc.Spec.DataSource = dataSource @@ -724,11 +725,13 @@ func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeCl func prepareSnapshotDataSourceForProvisioning( f *framework.Framework, config e2evolume.TestConfig, + perTestConfig *PerTestConfig, + pattern testpatterns.TestPattern, client clientset.Interface, dynamicClient dynamic.Interface, initClaim *v1.PersistentVolumeClaim, class *storagev1.StorageClass, - snapshotClass *unstructured.Unstructured, + sDriver SnapshottableTestDriver, mode v1.PersistentVolumeMode, injectContent string, ) (*v1.TypedLocalObjectReference, func()) { @@ -754,44 +757,30 @@ func prepareSnapshotDataSourceForProvisioning( } e2evolume.InjectContent(f, config, nil, "", tests) - ginkgo.By("[Initialize dataSource]creating a SnapshotClass") - snapshotClass, err = dynamicClient.Resource(SnapshotClassGVR).Create(context.TODO(), snapshotClass, metav1.CreateOptions{}) - framework.ExpectNoError(err) + snapshotResource := CreateSnapshotResource(sDriver, perTestConfig, pattern, updatedClaim.GetName(), updatedClaim.GetNamespace()) - ginkgo.By("[Initialize dataSource]creating a snapshot") - snapshot := getSnapshot(updatedClaim.Name, updatedClaim.Namespace, snapshotClass.GetName()) - snapshot, err = dynamicClient.Resource(SnapshotGVR).Namespace(updatedClaim.Namespace).Create(context.TODO(), snapshot, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout) - framework.ExpectNoError(err) - - ginkgo.By("[Initialize dataSource]checking the snapshot") - // Get new copy of the snapshot - snapshot, err = dynamicClient.Resource(SnapshotGVR).Namespace(snapshot.GetNamespace()).Get(context.TODO(), snapshot.GetName(), metav1.GetOptions{}) - framework.ExpectNoError(err) group := "snapshot.storage.k8s.io" dataSourceRef := &v1.TypedLocalObjectReference{ APIGroup: &group, Kind: "VolumeSnapshot", - Name: snapshot.GetName(), + Name: snapshotResource.Vs.GetName(), } cleanupFunc := func() { - framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName()) - err = dynamicClient.Resource(SnapshotGVR).Namespace(updatedClaim.Namespace).Delete(context.TODO(), snapshot.GetName(), metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - framework.Failf("Error deleting snapshot %q. Error: %v", snapshot.GetName(), err) - } - framework.Logf("deleting initClaim %q/%q", updatedClaim.Namespace, updatedClaim.Name) err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Delete(context.TODO(), updatedClaim.Name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { framework.Failf("Error deleting initClaim %q. Error: %v", updatedClaim.Name, err) } - framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName()) - framework.ExpectNoError(dynamicClient.Resource(SnapshotClassGVR).Delete(context.TODO(), snapshotClass.GetName(), metav1.DeleteOptions{})) + err = snapshotResource.CleanupResource() + framework.ExpectNoError(err) + + ginkgo.By("deleting StorageClass " + class.Name) + err = client.StorageV1().StorageClasses().Delete(context.TODO(), class.GetName(), metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Failf("Error deleting storage class %q. Error: %v", class.GetName(), err) + } } return dataSourceRef, cleanupFunc diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 1558fcdfc7b..b3765ff3198 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -71,11 +72,13 @@ func InitSnapshottableTestSuite() TestSuite { tsInfo: TestSuiteInfo{ Name: "snapshottable", TestPatterns: []testpatterns.TestPattern{ - testpatterns.DynamicSnapshot, + testpatterns.DynamicSnapshotDelete, + testpatterns.DynamicSnapshotRetain, }, SupportedSizeRange: e2evolume.SizeRange{ Min: "1Mi", }, + FeatureTag: "[Feature:VolumeSnapshotDataSource]", }, } } @@ -109,283 +112,135 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt // f must run inside an It or Context callback. f := framework.NewDefaultFramework("snapshotting") - init := func(l *snapshottableLocal) { - l.cs = f.ClientSet - l.dc = f.DynamicClient + ginkgo.Describe("volume snapshot controller", func() { + var ( + err error + config *PerTestConfig + driverCleanup func() + cleanupSteps []func() - // Now do the more expensive test initialization. - config, driverCleanup := driver.PrepareTest(f) - l.config = config - l.driverCleanup = driverCleanup + cs clientset.Interface + dc dynamic.Interface + pvc *v1.PersistentVolumeClaim + sc *storagev1.StorageClass + claimSize string + ) + init := func() { + cleanupSteps = make([]func(), 0) + // init snap class, create a source PV, PVC, Pod + cs = f.ClientSet + dc = f.DynamicClient - l.sc = dDriver.GetDynamicProvisionStorageClass(config, "") - if l.sc == nil { - framework.Failf("This driver should support dynamic provisioning") + // Now do the more expensive test initialization. + config, driverCleanup = driver.PrepareTest(f) + cleanupSteps = append(cleanupSteps, driverCleanup) + + volumeResource := CreateVolumeResource(dDriver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange) + + pvc = volumeResource.Pvc + sc = volumeResource.Sc + claimSize = pvc.Spec.Resources.Requests.Storage().String() + cleanupSteps = append(cleanupSteps, func() { + framework.ExpectNoError(volumeResource.CleanupResource()) + }) + + ginkgo.By("starting a pod to use the claim") + command := "echo 'hello world' > /mnt/test/data" + + RunInPodWithVolume(cs, pvc.Namespace, pvc.Name, "pvc-snapshottable-tester", command, config.ClientNodeSelection) + + err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, pvc.Namespace, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout) + framework.ExpectNoError(err) + + ginkgo.By("checking the claim") + // Get new copy of the claim + pvc, err = cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + // Get the bound PV + ginkgo.By("checking the PV") + _, err = cs.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{}) + framework.ExpectNoError(err) } - testVolumeSizeRange := s.GetTestSuiteInfo().SupportedSizeRange - driverVolumeSizeRange := dDriver.GetDriverInfo().SupportedSizeRange - claimSize, err := getSizeRangesIntersection(testVolumeSizeRange, driverVolumeSizeRange) - framework.ExpectNoError(err, "determine intersection of test size range %+v and driver size range %+v", testVolumeSizeRange, driverVolumeSizeRange) - l.pvc = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ - ClaimSize: claimSize, - StorageClassName: &(l.sc.Name), - }, config.Framework.Namespace.Name) - framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", l.sc, l.pvc) + cleanup := func() { + // Don't register an AfterEach then a cleanup step because the order + // of execution will do the AfterEach first then the cleanup step. + // Also AfterEach cleanup registration is not fine grained enough + // Adding to the cleanup steps allows you to register cleanup only when it is needed + // Ideally we could replace this with https://golang.org/pkg/testing/#T.Cleanup - ginkgo.By("creating a StorageClass " + l.sc.Name) - l.sc, err = l.cs.StorageV1().StorageClasses().Create(context.TODO(), l.sc, metav1.CreateOptions{}) - framework.ExpectNoError(err) - l.cleanupSteps = append(l.cleanupSteps, func() { - framework.Logf("deleting storage class %s", l.sc.Name) - framework.ExpectNoError(l.cs.StorageV1().StorageClasses().Delete(context.TODO(), l.sc.Name, metav1.DeleteOptions{})) - }) - - ginkgo.By("creating a claim") - l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.pvc.Namespace).Create(context.TODO(), l.pvc, metav1.CreateOptions{}) - framework.ExpectNoError(err) - l.cleanupSteps = append(l.cleanupSteps, func() { - framework.Logf("deleting claim %q/%q", l.pvc.Namespace, l.pvc.Name) - // typically this claim has already been deleted - err = l.cs.CoreV1().PersistentVolumeClaims(l.pvc.Namespace).Delete(context.TODO(), l.pvc.Name, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - framework.Failf("Error deleting claim %q. Error: %v", l.pvc.Name, err) + // Depending on how far the test executed, cleanup accordingly + // Execute in reverse order, similar to defer stack + for i := len(cleanupSteps) - 1; i >= 0; i-- { + err := tryFunc(cleanupSteps[i]) + framework.ExpectNoError(err, "while running cleanup steps") } - }) - ginkgo.By("starting a pod to use the claim") - command := "echo 'hello world' > /mnt/test/data" - l.pod = StartInPodWithVolume(l.cs, l.pvc.Namespace, l.pvc.Name, "pvc-snapshottable-tester", command, config.ClientNodeSelection) - l.cleanupSteps = append(l.cleanupSteps, func() { - StopPod(l.cs, l.pod) - }) - - err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, l.cs, l.pvc.Namespace, l.pvc.Name, framework.Poll, framework.ClaimProvisionTimeout) - framework.ExpectNoError(err) - - ginkgo.By("checking the claim") - // Get new copy of the claim - l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.pvc.Namespace).Get(context.TODO(), l.pvc.Name, metav1.GetOptions{}) - framework.ExpectNoError(err) - - // Get the bound PV - _, err = l.cs.CoreV1().PersistentVolumes().Get(context.TODO(), l.pvc.Spec.VolumeName, metav1.GetOptions{}) - framework.ExpectNoError(err) - } - cleanup := func(l *snapshottableLocal) { - // Depending on how far the test executed, cleanup accordingly - // Execute in reverse order, similar to defer stack - for i := len(l.cleanupSteps) - 1; i >= 0; i-- { - err := tryFunc(l.cleanupSteps[i]) - framework.ExpectNoError(err, "while running cleanup steps") } - - // All tests will require these driver cleanup tests - err := tryFunc(l.driverCleanup) - l.driverCleanup = nil - framework.ExpectNoError(err, "while cleaning up driver") - } - - ginkgo.It("should create snapshot with delete policy [Feature:VolumeSnapshotDataSource]", func() { - l := &snapshottableLocal{} - - init(l) - defer cleanup(l) - - TestSnapshottable(l, SnapshotClassTest{ - DeletionPolicy: "Delete", + ginkgo.BeforeEach(func() { + init() }) - TestSnapshotDeleted(l, SnapshotClassTest{ - DeletionPolicy: "Delete", + ginkgo.AfterEach(func() { + cleanup() }) - }) + ginkgo.Context("", func() { + var ( + vs *unstructured.Unstructured + vscontent *unstructured.Unstructured + vsc *unstructured.Unstructured + ) - ginkgo.It("should not delete snapshot with retain policy [Feature:VolumeSnapshotDataSource]", func() { - l := &snapshottableLocal{} + ginkgo.BeforeEach(func() { + sr := CreateSnapshotResource(sDriver, config, pattern, pvc.GetName(), pvc.GetNamespace()) + vs = sr.Vs + vscontent = sr.Vscontent + vsc = sr.Vsclass + cleanupSteps = append(cleanupSteps, func() { + framework.ExpectNoError(sr.CleanupResource()) + }) + }) - init(l) - defer cleanup(l) + ginkgo.It("should delete the VolumeSnapshotContent according to its deletion policy", func() { + DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + switch pattern.SnapshotDeletionPolicy { + case testpatterns.DeleteSnapshot: + ginkgo.By("checking the SnapshotContent has been deleted") + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + case testpatterns.RetainSnapshot: + ginkgo.By("checking the SnapshotContent has not been deleted") + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), 1*time.Second /* poll */, 30*time.Second /* timeout */) + framework.ExpectError(err) + } + }) + ginkgo.It("should create snapshot objects correctly", func() { + ginkgo.By("checking the snapshot") + // Get new copy of the snapshot + vs, err = dc.Resource(SnapshotGVR).Namespace(vs.GetNamespace()).Get(context.TODO(), vs.GetName(), metav1.GetOptions{}) + framework.ExpectNoError(err) + + // Get the bound snapshotContent + snapshotStatus := vs.Object["status"].(map[string]interface{}) + snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) + vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + snapshotContentSpec := vscontent.Object["spec"].(map[string]interface{}) + volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{}) + + // Check SnapshotContent properties + ginkgo.By("checking the SnapshotContent") + framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], vsc.GetName()) + framework.ExpectEqual(volumeSnapshotRef["name"], vs.GetName()) + framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace()) + }) - TestSnapshottable(l, SnapshotClassTest{ - DeletionPolicy: "Retain", - }) - TestSnapshotDeleted(l, SnapshotClassTest{ - DeletionPolicy: "Retain", }) }) } -// snapshottableLocal is used to keep the current state of a snapshottable -// test, associated objects, and cleanup steps. -type snapshottableLocal struct { - config *PerTestConfig - driverCleanup func() - cleanupSteps []func() - - cs clientset.Interface - dc dynamic.Interface - pvc *v1.PersistentVolumeClaim - sc *storagev1.StorageClass - pod *v1.Pod - vsc *unstructured.Unstructured - vs *unstructured.Unstructured - vscontent *unstructured.Unstructured -} - -// SnapshotClassTest represents parameters to be used by snapshot tests. -// Not all parameters are used by all tests. -type SnapshotClassTest struct { - DeletionPolicy string -} - -// TestSnapshottable tests volume snapshots based on a given SnapshotClassTest -func TestSnapshottable(l *snapshottableLocal, sct SnapshotClassTest) { - var err error - - ginkgo.By("creating a SnapshotClass") - l.vsc = sDriver.GetSnapshotClass(l.config) - if l.vsc == nil { - framework.Failf("Failed to get snapshot class based on test config") - } - l.vsc.Object["deletionPolicy"] = sct.DeletionPolicy - l.vsc, err = l.dc.Resource(SnapshotClassGVR).Create(context.TODO(), l.vsc, metav1.CreateOptions{}) - framework.ExpectNoError(err) - defer func() { - framework.Logf("deleting SnapshotClass %s", l.vsc.GetName()) - l.dc.Resource(SnapshotClassGVR).Delete(context.TODO(), l.vsc.GetName(), metav1.DeleteOptions{}) - }() - l.vsc, err = l.dc.Resource(SnapshotClassGVR).Namespace(l.vsc.GetNamespace()).Get(context.TODO(), l.vsc.GetName(), metav1.GetOptions{}) - framework.ExpectNoError(err) - - ginkgo.By("creating a snapshot") - l.vs = getSnapshot(l.pvc.Name, l.pvc.Namespace, l.vsc.GetName()) - - l.vs, err = l.dc.Resource(SnapshotGVR).Namespace(l.vs.GetNamespace()).Create(context.TODO(), l.vs, metav1.CreateOptions{}) - framework.ExpectNoError(err) - defer func() { - framework.Logf("deleting snapshot %q/%q", l.vs.GetNamespace(), l.vs.GetName()) - // typically this snapshot has already been deleted - err = l.dc.Resource(SnapshotGVR).Namespace(l.vs.GetNamespace()).Delete(context.TODO(), l.vs.GetName(), metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - framework.Failf("Error deleting snapshot %q. Error: %v", l.pvc.Name, err) - } - }() - err = WaitForSnapshotReady(l.dc, l.vs.GetNamespace(), l.vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout) - framework.ExpectNoError(err) - - ginkgo.By("checking the snapshot") - // Get new copy of the snapshot - l.vs, err = l.dc.Resource(SnapshotGVR).Namespace(l.vs.GetNamespace()).Get(context.TODO(), l.vs.GetName(), metav1.GetOptions{}) - framework.ExpectNoError(err) - - // Get the bound snapshotContent - snapshotStatus := l.vs.Object["status"].(map[string]interface{}) - snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) - l.vscontent, err = l.dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) - framework.ExpectNoError(err) - - snapshotContentSpec := l.vscontent.Object["spec"].(map[string]interface{}) - volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{}) - - // Check SnapshotContent properties - ginkgo.By("checking the SnapshotContent") - framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], l.vsc.GetName()) - framework.ExpectEqual(volumeSnapshotRef["name"], l.vs.GetName()) - framework.ExpectEqual(volumeSnapshotRef["namespace"], l.vs.GetNamespace()) -} - -// TestSnapshotDeleted tests the results of deleting a VolumeSnapshot -// depending on the deletion policy currently set. -func TestSnapshotDeleted(l *snapshottableLocal, sct SnapshotClassTest) { - var err error - - ginkgo.By("creating a SnapshotClass") - l.vsc = sDriver.GetSnapshotClass(l.config) - if l.vsc == nil { - framework.Failf("Failed to get snapshot class based on test config") - } - l.vsc.Object["deletionPolicy"] = sct.DeletionPolicy - l.vsc, err = l.dc.Resource(SnapshotClassGVR).Create(context.TODO(), l.vsc, metav1.CreateOptions{}) - framework.ExpectNoError(err) - defer func() { - framework.Logf("deleting SnapshotClass %s", l.vsc.GetName()) - l.dc.Resource(SnapshotClassGVR).Delete(context.TODO(), l.vsc.GetName(), metav1.DeleteOptions{}) - }() - l.vsc, err = l.dc.Resource(SnapshotClassGVR).Namespace(l.vsc.GetNamespace()).Get(context.TODO(), l.vsc.GetName(), metav1.GetOptions{}) - framework.ExpectNoError(err) - - ginkgo.By("creating a snapshot to delete") - l.vs = getSnapshot(l.pvc.Name, l.pvc.Namespace, l.vsc.GetName()) - - l.vs, err = l.dc.Resource(SnapshotGVR).Namespace(l.vs.GetNamespace()).Create(context.TODO(), l.vs, metav1.CreateOptions{}) - framework.ExpectNoError(err) - defer func() { - framework.Logf("deleting snapshot %q/%q", l.vs.GetNamespace(), l.vs.GetName()) - // typically this snapshot has already been deleted - err = l.dc.Resource(SnapshotGVR).Namespace(l.vs.GetNamespace()).Delete(context.TODO(), l.vs.GetName(), metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - framework.Failf("Error deleting snapshot %q. Error: %v", l.pvc.Name, err) - } - }() - err = WaitForSnapshotReady(l.dc, l.vs.GetNamespace(), l.vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout) - framework.ExpectNoError(err) - - ginkgo.By("get the snapshot to delete") - l.vs, err = l.dc.Resource(SnapshotGVR).Namespace(l.vs.GetNamespace()).Get(context.TODO(), l.vs.GetName(), metav1.GetOptions{}) - framework.ExpectNoError(err) - snapshotStatus := l.vs.Object["status"].(map[string]interface{}) - snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) - framework.Logf("received snapshotStatus %v", snapshotStatus) - framework.Logf("snapshotContentName %s", snapshotContentName) - l.vscontent, err = l.dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) - framework.ExpectNoError(err) - - ginkgo.By("deleting the snapshot") - err = l.dc.Resource(SnapshotGVR).Namespace(l.vs.GetNamespace()).Delete(context.TODO(), l.vs.GetName(), metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - framework.Failf("Error deleting snapshot %s in namespace %s. Error: %v", l.vs.GetName(), l.vs.GetNamespace(), err) - } - - ginkgo.By("checking the Snapshot has been deleted") - err = utils.WaitForGVRDeletion(l.dc, SnapshotGVR, l.sc.Name, framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err) - - if sct.DeletionPolicy == "Delete" { - ginkgo.By("checking the SnapshotContent has been deleted") - err = utils.WaitForGVRDeletion(l.dc, SnapshotContentGVR, snapshotContentName, framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err) - } else if sct.DeletionPolicy == "Retain" { - ginkgo.By("checking the SnapshotContent has not been deleted") - err = utils.WaitForGVRDeletion(l.dc, SnapshotContentGVR, snapshotContentName, 1*time.Second /* poll */, 30*time.Second /* timeout */) - framework.ExpectError(err) // should fail deletion check - - // The purpose of this block is to prevent physical snapshotContent leaks. - // We must update the SnapshotContent to have Delete Deletion policy, - // or else the physical snapshot content will be leaked. - ginkgo.By("get the latest copy of volume snapshot content") - snapshotContent, err := l.dc.Resource(SnapshotContentGVR).Get(context.TODO(), l.vscontent.GetName(), metav1.GetOptions{}) - framework.ExpectNoError(err) - - ginkgo.By("updating the the SnapshotContent to have Delete Deletion policy") - snapshotContent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Delete" - l.vscontent, err = l.dc.Resource(SnapshotContentGVR).Update(context.TODO(), snapshotContent, metav1.UpdateOptions{}) - framework.ExpectNoError(err) - - ginkgo.By("manually deleting the SnapshotContent") - err = l.dc.Resource(SnapshotContentGVR).Delete(context.TODO(), snapshotContent.GetName(), metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - framework.Failf("Error deleting snapshot content %q. Error: %v", snapshotContent.GetName(), err) - } - - ginkgo.By("checking the SnapshotContent has been deleted") - err = utils.WaitForGVRDeletion(l.dc, SnapshotContentGVR, snapshotContentName, framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err) - } else { - framework.Failf("Invalid test config. DeletionPolicy should be either Delete or Retain. DeletionPolicy: %v", sct.DeletionPolicy) - } -} - // WaitForSnapshotReady waits for a VolumeSnapshot to be ready to use or until timeout occurs, whichever comes first. func WaitForSnapshotReady(c dynamic.Interface, ns string, snapshotName string, poll, timeout time.Duration) error { framework.Logf("Waiting up to %v for VolumeSnapshot %s to become ready", timeout, snapshotName) @@ -416,3 +271,175 @@ func WaitForSnapshotReady(c dynamic.Interface, ns string, snapshotName string, p return fmt.Errorf("VolumeSnapshot %s is not ready within %v", snapshotName, timeout) } + +// DeleteAndWaitSnapshot deletes a VolumeSnapshot and waits for it to be deleted or until timeout occurs, whichever comes first +func DeleteAndWaitSnapshot(dc dynamic.Interface, ns string, snapshotName string, poll, timeout time.Duration) error { + var err error + ginkgo.By("deleting the snapshot") + err = dc.Resource(SnapshotGVR).Namespace(ns).Delete(context.TODO(), snapshotName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Failf("Error deleting snapshot %s in namespace %s. Error: %v", snapshotName, ns, err) + } + + ginkgo.By("checking the Snapshot has been deleted") + err = utils.WaitForGVRDeletion(dc, SnapshotGVR, snapshotName, poll, timeout) + + return err +} + +// SnapshotResource represents a snapshot class, a snapshot and its bound snapshot contents for a specific test case +type SnapshotResource struct { + Config *PerTestConfig + Pattern testpatterns.TestPattern + + Vs *unstructured.Unstructured + Vscontent *unstructured.Unstructured + Vsclass *unstructured.Unstructured +} + +// CreateSnapshotResource creates a snapshot resource for the current test. It knows how to deal with +// different test pattern snapshot provisioning and deletion policy +func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConfig, pattern testpatterns.TestPattern, pvcName string, pvcNamespace string) *SnapshotResource { + var err error + r := SnapshotResource{ + Config: config, + Pattern: pattern, + } + dc := r.Config.Framework.DynamicClient + + ginkgo.By("creating a SnapshotClass") + r.Vsclass = sDriver.GetSnapshotClass(config) + if r.Vsclass == nil { + framework.Failf("Failed to get snapshot class based on test config") + } + r.Vsclass.Object["deletionPolicy"] = pattern.SnapshotDeletionPolicy + + r.Vsclass, err = dc.Resource(SnapshotClassGVR).Create(context.TODO(), r.Vsclass, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + r.Vsclass, err = dc.Resource(SnapshotClassGVR).Namespace(r.Vsclass.GetNamespace()).Get(context.TODO(), r.Vsclass.GetName(), metav1.GetOptions{}) + framework.ExpectNoError(err) + + switch pattern.SnapshotType { + case testpatterns.DynamicCreatedSnapshot: + ginkgo.By("creating a VolumeSnapshot") + // prepare a dynamically provisioned volume snapshot with certain data + r.Vs = getSnapshot(pvcName, pvcNamespace, r.Vsclass.GetName()) + + r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + err = WaitForSnapshotReady(dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout) + framework.ExpectNoError(err) + + r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(context.TODO(), r.Vs.GetName(), metav1.GetOptions{}) + + snapshotStatus := r.Vs.Object["status"].(map[string]interface{}) + snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) + framework.Logf("received snapshotStatus %v", snapshotStatus) + framework.Logf("snapshotContentName %s", snapshotContentName) + framework.ExpectNoError(err) + + r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) + framework.ExpectNoError(err) + case testpatterns.PreprovisionedCreatedSnapshot: + // prepare a pre-provisioned VolumeSnapshotContent with certain data + // Because this could be run with an external CSI driver, we have no way + // to pre-provision the snapshot as we normally would using their API. + // We instead dynamically take a snapshot and create another snapshot using + // the first snapshot's snapshot handle. + ginkgo.Skip("Preprovisioned test not implemented") + ginkgo.By("taking a snapshot with deletion policy retain") + ginkgo.By("recording the volume handle and status.snapshotHandle") + ginkgo.By("deleting the snapshot and snapshot content") // TODO: test what happens when I have two snapshot content that refer to the same content + ginkgo.By("creating a snapshot content with the snapshot handle") + ginkgo.By("creating a snapshot with that snapshot content") + } + return &r +} + +// CleanupResource cleans up the snapshot resource and ignores not found errors +func (sr *SnapshotResource) CleanupResource() error { + var err error + var cleanupErrs []error + + dc := sr.Config.Framework.DynamicClient + + if sr.Vs != nil { + framework.Logf("deleting snapshot %q/%q", sr.Vs.GetNamespace(), sr.Vs.GetName()) + + sr.Vs, err = dc.Resource(SnapshotGVR).Namespace(sr.Vs.GetNamespace()).Get(context.TODO(), sr.Vs.GetName(), metav1.GetOptions{}) + switch { + case err == nil: + snapshotStatus := sr.Vs.Object["status"].(map[string]interface{}) + snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) + framework.Logf("received snapshotStatus %v", snapshotStatus) + framework.Logf("snapshotContentName %s", snapshotContentName) + + boundVsContent, err := dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) + switch { + case err == nil: + if boundVsContent.Object["spec"].(map[string]interface{})["deletionPolicy"] != "Delete" { + // The purpose of this block is to prevent physical snapshotContent leaks. + // We must update the SnapshotContent to have Delete Deletion policy, + // or else the physical snapshot content will be leaked. + boundVsContent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Delete" + boundVsContent, err = dc.Resource(SnapshotContentGVR).Update(context.TODO(), boundVsContent, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + } + err = dc.Resource(SnapshotGVR).Namespace(sr.Vs.GetNamespace()).Delete(context.TODO(), sr.Vs.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, boundVsContent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + case apierrors.IsNotFound(err): + // the volume snapshot is not bound to snapshot content yet + err = dc.Resource(SnapshotGVR).Namespace(sr.Vs.GetNamespace()).Delete(context.TODO(), sr.Vs.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = utils.WaitForGVRDeletion(dc, SnapshotGVR, sr.Vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + default: + cleanupErrs = append(cleanupErrs, err) + } + case apierrors.IsNotFound(err): + // Hope that the underlying snapshot content and resource is gone already + default: + cleanupErrs = append(cleanupErrs, err) + } + } + if sr.Vscontent != nil { + framework.Logf("deleting snapshot content %q/%q", sr.Vscontent.GetNamespace(), sr.Vscontent.GetName()) + + sr.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), sr.Vscontent.GetName(), metav1.GetOptions{}) + switch { + case err == nil: + if sr.Vscontent.Object["spec"].(map[string]interface{})["deletionPolicy"] != "Delete" { + // The purpose of this block is to prevent physical snapshotContent leaks. + // We must update the SnapshotContent to have Delete Deletion policy, + // or else the physical snapshot content will be leaked. + sr.Vscontent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Delete" + sr.Vscontent, err = dc.Resource(SnapshotContentGVR).Update(context.TODO(), sr.Vscontent, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + } + err = dc.Resource(SnapshotContentGVR).Namespace(sr.Vscontent.GetNamespace()).Delete(context.TODO(), sr.Vscontent.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, sr.Vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + case apierrors.IsNotFound(err): + // Hope the underlying physical snapshot resource has been deleted already + default: + cleanupErrs = append(cleanupErrs, err) + } + } + if sr.Vsclass != nil { + framework.Logf("deleting snapshot class %q/%q", sr.Vsclass.GetNamespace(), sr.Vsclass.GetName()) + // typically this snapshot class has already been deleted + err = dc.Resource(SnapshotClassGVR).Namespace(sr.Vsclass.GetNamespace()).Delete(context.TODO(), sr.Vsclass.GetName(), metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Failf("Error deleting snapshot class %q. Error: %v", sr.Vsclass.GetName(), err) + } + err = utils.WaitForGVRDeletion(dc, SnapshotClassGVR, sr.Vsclass.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + } + return utilerrors.NewAggregate(cleanupErrs) +} From 913f626adb6bc5925fa65fc58044ef4b8cbce2bf Mon Sep 17 00:00:00 2001 From: Andi Li Date: Wed, 8 Jul 2020 16:33:55 +0000 Subject: [PATCH 2/3] Add data semantics test for volume snapshots The test steps are as follows: 1. Write some data 2. Take a snapshot 3. Write more data 4. Create a new volume from snapshot 5. Validate data is the old data --- test/e2e/storage/testsuites/snapshottable.go | 47 ++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index b3765ff3198..efac710b55c 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" @@ -236,7 +237,53 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt framework.ExpectEqual(volumeSnapshotRef["name"], vs.GetName()) framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace()) }) + ginkgo.It("should restore from snapshot with saved data after modifying source data", func() { + var restoredPVC *v1.PersistentVolumeClaim + var restoredPod *v1.Pod + ginkgo.By("modifying the data in the source PVC") + command := "echo junkdata > /mnt/test/data" + RunInPodWithVolume(cs, pvc.Namespace, pvc.Name, "pvc-snapshottable-data-tester", command, config.ClientNodeSelection) + + ginkgo.By("creating a pvc from the snapshot") + restoredPVC = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ + ClaimSize: claimSize, + StorageClassName: &(sc.Name), + }, config.Framework.Namespace.Name) + + group := "snapshot.storage.k8s.io" + dataSourceRef := &v1.TypedLocalObjectReference{ + APIGroup: &group, + Kind: "VolumeSnapshot", + Name: vs.GetName(), + } + + restoredPVC.Spec.DataSource = dataSourceRef + + restoredPVC, err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Create(context.TODO(), restoredPVC, metav1.CreateOptions{}) + framework.ExpectNoError(err) + cleanupSteps = append(cleanupSteps, func() { + framework.Logf("deleting claim %q/%q", restoredPVC.Namespace, restoredPVC.Name) + // typically this claim has already been deleted + err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Delete(context.TODO(), restoredPVC.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Failf("Error deleting claim %q. Error: %v", restoredPVC.Name, err) + } + }) + + ginkgo.By("starting a pod to use the claim") + + restoredPod = StartInPodWithVolume(cs, restoredPVC.Namespace, restoredPVC.Name, "restored-pvc-tester", "sleep 300", config.ClientNodeSelection) + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(cs, restoredPod.Name, restoredPod.Namespace)) + cleanupSteps = append(cleanupSteps, func() { + StopPod(cs, restoredPod) + }) + + command = "cat /mnt/test/data" + actualData, err := utils.PodExec(f, restoredPod, command) + framework.ExpectNoError(err) + framework.ExpectEqual(actualData, "hello world") + }) }) }) } From b64f05b70ab0e4ef986d229ece4ac02566ff0fe5 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Wed, 8 Jul 2020 22:41:48 +0000 Subject: [PATCH 3/3] Change expected content to string constant. Cleanup DeleteAndWaitSnapshot Use the correct format specifier --- test/e2e/storage/testsuites/snapshottable.go | 28 ++++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index efac710b55c..7cba54f7b6b 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -120,11 +120,12 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt driverCleanup func() cleanupSteps []func() - cs clientset.Interface - dc dynamic.Interface - pvc *v1.PersistentVolumeClaim - sc *storagev1.StorageClass - claimSize string + cs clientset.Interface + dc dynamic.Interface + pvc *v1.PersistentVolumeClaim + sc *storagev1.StorageClass + claimSize string + originalMntTestData string ) init := func() { cleanupSteps = make([]func(), 0) @@ -146,7 +147,8 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt }) ginkgo.By("starting a pod to use the claim") - command := "echo 'hello world' > /mnt/test/data" + originalMntTestData = fmt.Sprintf("hello from %s namespace", pvc.GetNamespace()) + command := fmt.Sprintf("echo '%s' > /mnt/test/data", originalMntTestData) RunInPodWithVolume(cs, pvc.Namespace, pvc.Name, "pvc-snapshottable-tester", command, config.ClientNodeSelection) @@ -204,7 +206,9 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt }) ginkgo.It("should delete the VolumeSnapshotContent according to its deletion policy", func() { - DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + err = DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + switch pattern.SnapshotDeletionPolicy { case testpatterns.DeleteSnapshot: ginkgo.By("checking the SnapshotContent has been deleted") @@ -240,9 +244,11 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt ginkgo.It("should restore from snapshot with saved data after modifying source data", func() { var restoredPVC *v1.PersistentVolumeClaim var restoredPod *v1.Pod + modifiedMntTestData := fmt.Sprintf("modified data from %s namespace", pvc.GetNamespace()) ginkgo.By("modifying the data in the source PVC") - command := "echo junkdata > /mnt/test/data" + + command := fmt.Sprintf("echo '%s' > /mnt/test/data", modifiedMntTestData) RunInPodWithVolume(cs, pvc.Namespace, pvc.Name, "pvc-snapshottable-data-tester", command, config.ClientNodeSelection) ginkgo.By("creating a pvc from the snapshot") @@ -282,7 +288,7 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt command = "cat /mnt/test/data" actualData, err := utils.PodExec(f, restoredPod, command) framework.ExpectNoError(err) - framework.ExpectEqual(actualData, "hello world") + framework.ExpectEqual(actualData, originalMntTestData) }) }) }) @@ -324,8 +330,8 @@ func DeleteAndWaitSnapshot(dc dynamic.Interface, ns string, snapshotName string, var err error ginkgo.By("deleting the snapshot") err = dc.Resource(SnapshotGVR).Namespace(ns).Delete(context.TODO(), snapshotName, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - framework.Failf("Error deleting snapshot %s in namespace %s. Error: %v", snapshotName, ns, err) + if err != nil { + return err } ginkgo.By("checking the Snapshot has been deleted")