From ed851298312b03377242fd17ac7999b8d08e52c8 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Tue, 14 Jul 2020 22:28:15 +0000 Subject: [PATCH 01/11] Minor cleanup in snapshot test suite. --- test/e2e/storage/testsuites/snapshottable.go | 39 ++++++++++---------- test/e2e/storage/utils/utils.go | 27 +++++++++++++- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 06feb277bf3..15f4584d59f 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -137,14 +137,15 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt config, driverCleanup = driver.PrepareTest(f) cleanupSteps = append(cleanupSteps, driverCleanup) - volumeResource := CreateVolumeResource(dDriver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange) + var volumeResource *VolumeResource + cleanupSteps = append(cleanupSteps, func() { + framework.ExpectNoError(volumeResource.CleanupResource()) + }) + volumeResource = CreateVolumeResource(dDriver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange) pvc = volumeResource.Pvc sc = volumeResource.Sc claimSize = pvc.Spec.Resources.Requests.Storage().String() - cleanupSteps = append(cleanupSteps, func() { - framework.ExpectNoError(volumeResource.CleanupResource()) - }) ginkgo.By("starting a pod to use the claim") originalMntTestData = fmt.Sprintf("hello from %s namespace", pvc.GetNamespace()) @@ -196,13 +197,14 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt ) ginkgo.BeforeEach(func() { - sr := CreateSnapshotResource(sDriver, config, pattern, pvc.GetName(), pvc.GetNamespace()) - vs = sr.Vs - vscontent = sr.Vscontent - vsc = sr.Vsclass + var sr *SnapshotResource cleanupSteps = append(cleanupSteps, func() { framework.ExpectNoError(sr.CleanupResource()) }) + sr = CreateSnapshotResource(sDriver, config, pattern, pvc.GetName(), pvc.GetNamespace()) + vs = sr.Vs + vscontent = sr.Vscontent + vsc = sr.Vsclass }) ginkgo.It("should delete the VolumeSnapshotContent according to its deletion policy", func() { @@ -258,14 +260,13 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt }, config.Framework.Namespace.Name) group := "snapshot.storage.k8s.io" - dataSourceRef := &v1.TypedLocalObjectReference{ + + restoredPVC.Spec.DataSource = &v1.TypedLocalObjectReference{ APIGroup: &group, Kind: "VolumeSnapshot", Name: vs.GetName(), } - restoredPVC.Spec.DataSource = dataSourceRef - restoredPVC, err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Create(context.TODO(), restoredPVC, metav1.CreateOptions{}) framework.ExpectNoError(err) cleanupSteps = append(cleanupSteps, func() { @@ -280,10 +281,10 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt ginkgo.By("starting a pod to use the claim") restoredPod = StartInPodWithVolume(cs, restoredPVC.Namespace, restoredPVC.Name, "restored-pvc-tester", "sleep 300", config.ClientNodeSelection) - framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(cs, restoredPod.Name, restoredPod.Namespace)) cleanupSteps = append(cleanupSteps, func() { StopPod(cs, restoredPod) }) + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(cs, restoredPod.Name, restoredPod.Namespace)) command = "cat /mnt/test/data" actualData, err := utils.PodExec(f, restoredPod, command) @@ -335,7 +336,7 @@ func DeleteAndWaitSnapshot(dc dynamic.Interface, ns string, snapshotName string, } ginkgo.By("checking the Snapshot has been deleted") - err = utils.WaitForGVRDeletion(dc, SnapshotGVR, snapshotName, poll, timeout) + err = utils.WaitForNamespacedGVRDeletion(dc, SnapshotGVR, snapshotName, ns, poll, timeout) return err } @@ -370,7 +371,7 @@ func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConf r.Vsclass, err = dc.Resource(SnapshotClassGVR).Create(context.TODO(), r.Vsclass, metav1.CreateOptions{}) framework.ExpectNoError(err) - r.Vsclass, err = dc.Resource(SnapshotClassGVR).Namespace(r.Vsclass.GetNamespace()).Get(context.TODO(), r.Vsclass.GetName(), metav1.GetOptions{}) + r.Vsclass, err = dc.Resource(SnapshotClassGVR).Get(context.TODO(), r.Vsclass.GetName(), metav1.GetOptions{}) framework.ExpectNoError(err) switch pattern.SnapshotType { @@ -451,7 +452,7 @@ func (sr *SnapshotResource) CleanupResource() error { // the volume snapshot is not bound to snapshot content yet err = dc.Resource(SnapshotGVR).Namespace(sr.Vs.GetNamespace()).Delete(context.TODO(), sr.Vs.GetName(), metav1.DeleteOptions{}) framework.ExpectNoError(err) - err = utils.WaitForGVRDeletion(dc, SnapshotGVR, sr.Vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + err = utils.WaitForNamespacedGVRDeletion(dc, SnapshotGVR, sr.Vs.GetName(), sr.Vs.GetNamespace(), framework.Poll, framework.SnapshotDeleteTimeout) framework.ExpectNoError(err) default: cleanupErrs = append(cleanupErrs, err) @@ -463,7 +464,7 @@ func (sr *SnapshotResource) CleanupResource() error { } } if sr.Vscontent != nil { - framework.Logf("deleting snapshot content %q/%q", sr.Vscontent.GetNamespace(), sr.Vscontent.GetName()) + framework.Logf("deleting snapshot content %q", sr.Vscontent.GetName()) sr.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), sr.Vscontent.GetName(), metav1.GetOptions{}) switch { @@ -476,7 +477,7 @@ func (sr *SnapshotResource) CleanupResource() error { sr.Vscontent, err = dc.Resource(SnapshotContentGVR).Update(context.TODO(), sr.Vscontent, metav1.UpdateOptions{}) framework.ExpectNoError(err) } - err = dc.Resource(SnapshotContentGVR).Namespace(sr.Vscontent.GetNamespace()).Delete(context.TODO(), sr.Vscontent.GetName(), metav1.DeleteOptions{}) + err = dc.Resource(SnapshotContentGVR).Delete(context.TODO(), sr.Vscontent.GetName(), metav1.DeleteOptions{}) framework.ExpectNoError(err) err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, sr.Vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) @@ -488,9 +489,9 @@ func (sr *SnapshotResource) CleanupResource() error { } } if sr.Vsclass != nil { - framework.Logf("deleting snapshot class %q/%q", sr.Vsclass.GetNamespace(), sr.Vsclass.GetName()) + framework.Logf("deleting snapshot class %q", sr.Vsclass.GetName()) // typically this snapshot class has already been deleted - err = dc.Resource(SnapshotClassGVR).Namespace(sr.Vsclass.GetNamespace()).Delete(context.TODO(), sr.Vsclass.GetName(), metav1.DeleteOptions{}) + err = dc.Resource(SnapshotClassGVR).Delete(context.TODO(), sr.Vsclass.GetName(), metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { framework.Failf("Error deleting snapshot class %q. Error: %v", sr.Vsclass.GetName(), err) } diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index e6ac7027b30..7ee13f4f9ac 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -727,7 +727,7 @@ func CreateDriverNamespace(f *framework.Framework) *v1.Namespace { return namespace } -// WaitForGVRDeletion waits until an object has been deleted +// WaitForGVRDeletion waits until a non-namespaced object has been deleted func WaitForGVRDeletion(c dynamic.Interface, gvr schema.GroupVersionResource, objectName string, poll, timeout time.Duration) error { framework.Logf("Waiting up to %v for %s %s to be deleted", timeout, gvr.Resource, objectName) @@ -737,7 +737,7 @@ func WaitForGVRDeletion(c dynamic.Interface, gvr schema.GroupVersionResource, ob framework.Logf("%s %v is not found and has been deleted", gvr.Resource, objectName) return true } else if err != nil { - framework.Logf("Get $s %v returned an error: %v", objectName, err.Error()) + framework.Logf("Get %s returned an error: %v", objectName, err.Error()) } else { framework.Logf("%s %v has been found and is not deleted", gvr.Resource, objectName) } @@ -750,6 +750,29 @@ func WaitForGVRDeletion(c dynamic.Interface, gvr schema.GroupVersionResource, ob return fmt.Errorf("%s %s is not deleted within %v", gvr.Resource, objectName, timeout) } +// WaitForNamespacedGVRDeletion waits until a namespaced object has been deleted +func WaitForNamespacedGVRDeletion(c dynamic.Interface, gvr schema.GroupVersionResource, ns, objectName string, poll, timeout time.Duration) error { + framework.Logf("Waiting up to %v for %s %s to be deleted", timeout, gvr.Resource, objectName) + + if successful := WaitUntil(poll, timeout, func() bool { + _, err := c.Resource(gvr).Namespace(ns).Get(context.TODO(), objectName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + framework.Logf("%s %s is not found in namespace %s and has been deleted", gvr.Resource, objectName, ns) + return true + } else if err != nil { + framework.Logf("Get %s in namespace %s returned an error: %v", objectName, ns, err.Error()) + } else { + framework.Logf("%s %s has been found in namespace %s and is not deleted", gvr.Resource, objectName, ns) + } + + return false + }); successful { + return nil + } + + return fmt.Errorf("%s %s in namespace %s is not deleted within %v", gvr.Resource, objectName, ns, timeout) +} + // WaitUntil runs checkDone until a timeout is reached func WaitUntil(poll, timeout time.Duration, checkDone func() bool) bool { for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) { From e78aee99654d571bb2d5deb3be7d5935de396be1 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Tue, 14 Jul 2020 22:29:07 +0000 Subject: [PATCH 02/11] Add pre provisioned snapshot tests Make pre provisioned snapshots using CSI driver by 1. Take a dynamic snapshot with retain policy 2. Delete the dynamic snapshot and content 3. Create a preprovisioned snapshot with snapshotHandle This commit adds a preprovisiond test pattern, all snapshots made using create snapshot resource become prepv snapshots. All exisitng test cases now run again with prepv snapshots. --- test/e2e/storage/external/external.go | 4 ++ test/e2e/storage/testpatterns/testpattern.go | 4 ++ test/e2e/storage/testsuites/base.go | 59 ++++++++++++++++ test/e2e/storage/testsuites/snapshottable.go | 73 ++++++++++++++++++-- 4 files changed, 133 insertions(+), 7 deletions(-) diff --git a/test/e2e/storage/external/external.go b/test/e2e/storage/external/external.go index b21a6f88094..ac0eb515a22 100644 --- a/test/e2e/storage/external/external.go +++ b/test/e2e/storage/external/external.go @@ -255,6 +255,10 @@ func (d *driverDefinition) SkipUnsupportedTest(pattern testpatterns.TestPattern) if d.SnapshotClass.FromName || d.SnapshotClass.FromFile != "" || d.SnapshotClass.FromExistingClassName != "" { supported = true } + case testpatterns.PreprovisionedCreatedSnapshot: + if d.SnapshotClass.FromName || d.SnapshotClass.FromFile != "" || d.SnapshotClass.FromExistingClassName != "" { + supported = true + } } if !supported { e2eskipper.Skipf("Driver %q does not support snapshot type %q - skipping", d.DriverInfo.Name, pattern.SnapshotType) diff --git a/test/e2e/storage/testpatterns/testpattern.go b/test/e2e/storage/testpatterns/testpattern.go index cd971b94fd7..f6057999e16 100644 --- a/test/e2e/storage/testpatterns/testpattern.go +++ b/test/e2e/storage/testpatterns/testpattern.go @@ -70,6 +70,10 @@ var ( RetainSnapshot TestSnapshotDeletionPolicy = "Retain" ) +func (t TestSnapshotDeletionPolicy) String() string { + return string(t) +} + // TestPattern represents a combination of parameters to be tested in a TestSuite type TestPattern struct { Name string // Name of TestPattern diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 7519d0a0806..079d07a9b2d 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -18,6 +18,7 @@ package testsuites import ( "context" + "crypto/sha1" "flag" "fmt" "math" @@ -571,6 +572,64 @@ func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.U return snapshot } +func getPreProvisionedSnapshot(snapshotContentName, ns, snapshotHandle string) *unstructured.Unstructured { + snapshot := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "VolumeSnapshot", + "apiVersion": snapshotAPIVersion, + "metadata": map[string]interface{}{ + "name": getPreProvisionedSnapshotName(snapshotHandle), + "namespace": ns, + }, + "spec": map[string]interface{}{ + "source": map[string]interface{}{ + "volumeSnapshotContentName": snapshotContentName, + }, + }, + }, + } + + return snapshot +} +func getPreProvisionedSnapshotContent(snapshotName, snapshotNamespace, snapshotHandle, deletionPolicy, csiDriverName, volumeSnapshotClassName string) *unstructured.Unstructured { + snapshotContent := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "VolumeSnapshotContent", + "apiVersion": snapshotAPIVersion, + "metadata": map[string]interface{}{ + "name": getPreProvisionedSnapshotContentName(snapshotHandle), + }, + "spec": map[string]interface{}{ + "source": map[string]interface{}{ + "snapshotHandle": snapshotHandle, + }, + "volumeSnapshotRef": map[string]interface{}{ + "name": snapshotName, + "namespace": snapshotNamespace, + }, + "driver": csiDriverName, + "deletionPolicy": deletionPolicy, + "volumeSnapshotClassName": volumeSnapshotClassName, + }, + }, + } + + return snapshotContent +} + +func genShortHash(s string) string { + h := sha1.New() + h.Write([]byte(s)) + bs := h.Sum(nil) + return fmt.Sprintf("%x", bs)[:7] +} +func getPreProvisionedSnapshotContentName(snapshotHandle string) string { + return fmt.Sprintf("pre-provisioned-snapcontent-%s", genShortHash(snapshotHandle)) +} + +func getPreProvisionedSnapshotName(snapshotHandle string) string { + return fmt.Sprintf("pre-provisioned-snapshot-%s", genShortHash(snapshotHandle)) +} // StartPodLogs begins capturing log output and events from current // and future pods running in the namespace of the framework. That diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 15f4584d59f..59cac10d035 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -75,6 +75,8 @@ func InitSnapshottableTestSuite() TestSuite { TestPatterns: []testpatterns.TestPattern{ testpatterns.DynamicSnapshotDelete, testpatterns.DynamicSnapshotRetain, + testpatterns.PreprovisionedSnapshotDelete, + testpatterns.PreprovisionedSnapshotRetain, }, SupportedSizeRange: e2evolume.SizeRange{ Min: "1Mi", @@ -94,7 +96,6 @@ func (s *snapshottableTestSuite) SkipRedundantSuite(driver TestDriver, pattern t func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatterns.TestPattern) { ginkgo.BeforeEach(func() { // Check preconditions. - framework.ExpectEqual(pattern.SnapshotType, testpatterns.DynamicCreatedSnapshot) dInfo := driver.GetDriverInfo() ok := false sDriver, ok = driver.(SnapshottableTestDriver) @@ -355,6 +356,10 @@ type SnapshotResource struct { // different test pattern snapshot provisioning and deletion policy func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConfig, pattern testpatterns.TestPattern, pvcName string, pvcNamespace string) *SnapshotResource { var err error + if pattern.SnapshotType != testpatterns.DynamicCreatedSnapshot || pattern.SnapshotType != testpatterns.PreprovisionedCreatedSnapshot { + err = fmt.Errorf("SnapshotType must be set to either DynamicCreatedSnapshot or PreprovisionedCreatedSnapshot") + framework.ExpectNoError(err) + } r := SnapshotResource{ Config: config, Pattern: pattern, @@ -366,7 +371,7 @@ func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConf if r.Vsclass == nil { framework.Failf("Failed to get snapshot class based on test config") } - r.Vsclass.Object["deletionPolicy"] = pattern.SnapshotDeletionPolicy + r.Vsclass.Object["deletionPolicy"] = pattern.SnapshotDeletionPolicy.String() r.Vsclass, err = dc.Resource(SnapshotClassGVR).Create(context.TODO(), r.Vsclass, metav1.CreateOptions{}) framework.ExpectNoError(err) @@ -402,14 +407,68 @@ func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConf // to pre-provision the snapshot as we normally would using their API. // We instead dynamically take a snapshot and create another snapshot using // the first snapshot's snapshot handle. - ginkgo.Skip("Preprovisioned test not implemented") - ginkgo.By("taking a snapshot with deletion policy retain") - ginkgo.By("recording the volume handle and status.snapshotHandle") + ginkgo.By("taking a dynamic snapshot") + r.Vs = getSnapshot(pvcName, pvcNamespace, r.Vsclass.GetName()) + r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + err = WaitForSnapshotReady(dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout) + framework.ExpectNoError(err) + + r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(context.TODO(), r.Vs.GetName(), metav1.GetOptions{}) + + snapshotStatus := r.Vs.Object["status"].(map[string]interface{}) + snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) + framework.Logf("received snapshotStatus %v", snapshotStatus) + framework.Logf("snapshotContentName %s", snapshotContentName) + framework.ExpectNoError(err) + + ginkgo.By("updating the snapshot content deletion policy to retain") + r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) + framework.ExpectNoError(err) + r.Vscontent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Retain" + + r.Vscontent, err = dc.Resource(SnapshotContentGVR).Update(context.TODO(), r.Vscontent, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("recording the volume handle and snapshotHandle") + snapshotHandle := r.Vscontent.Object["status"].(map[string]interface{})["snapshotHandle"].(string) + framework.Logf("Recording snapshot handle: %s", snapshotHandle) + csiDriverName := r.Vsclass.Object["driver"].(string) + ginkgo.By("deleting the snapshot and snapshot content") // TODO: test what happens when I have two snapshot content that refer to the same content + err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Delete(context.TODO(), r.Vs.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("checking the Snapshot has been deleted") + err = utils.WaitForNamespacedGVRDeletion(dc, SnapshotGVR, r.Vs.GetName(), r.Vs.GetNamespace(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + + err = dc.Resource(SnapshotContentGVR).Delete(context.TODO(), r.Vscontent.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("checking the Snapshot content has been deleted") + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, r.Vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + ginkgo.By("creating a snapshot content with the snapshot handle") + r.Vscontent = getPreProvisionedSnapshotContent(getPreProvisionedSnapshotName(snapshotHandle), pvcNamespace, snapshotHandle, pattern.SnapshotDeletionPolicy.String(), csiDriverName, r.Vsclass.GetName()) + r.Vscontent, err = dc.Resource(SnapshotContentGVR).Create(context.TODO(), r.Vscontent, metav1.CreateOptions{}) + framework.ExpectNoError(err) + ginkgo.By("creating a snapshot with that snapshot content") - default: - err = fmt.Errorf("SnapshotType must be set to either DynamicCreatedSnapshot or PreprovisionedCreatedSnapshot") + r.Vs = getPreProvisionedSnapshot(getPreProvisionedSnapshotContentName(snapshotHandle), pvcNamespace, snapshotHandle) + r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + err = WaitForSnapshotReady(dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout) + framework.ExpectNoError(err) + + ginkgo.By("getting the snapshot and snapshot content") + r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(context.TODO(), r.Vs.GetName(), metav1.GetOptions{}) + framework.ExpectNoError(err) + + r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), r.Vscontent.GetName(), metav1.GetOptions{}) framework.ExpectNoError(err) } return &r From 42b2d1c52ca39d85d643488af9706e08c0acf79a Mon Sep 17 00:00:00 2001 From: Andi Li Date: Fri, 17 Jul 2020 15:08:39 +0000 Subject: [PATCH 03/11] Combine two tests with overlap to reduce overall snapshot test suite time --- test/e2e/storage/testsuites/snapshottable.go | 33 ++++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 59cac10d035..c6bb19c0f0f 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -207,23 +207,7 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt vscontent = sr.Vscontent vsc = sr.Vsclass }) - - ginkgo.It("should delete the VolumeSnapshotContent according to its deletion policy", func() { - err = DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err) - - switch pattern.SnapshotDeletionPolicy { - case testpatterns.DeleteSnapshot: - ginkgo.By("checking the SnapshotContent has been deleted") - err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err) - case testpatterns.RetainSnapshot: - ginkgo.By("checking the SnapshotContent has not been deleted") - err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), 1*time.Second /* poll */, 30*time.Second /* timeout */) - framework.ExpectError(err) - } - }) - ginkgo.It("should create snapshot objects correctly", func() { + ginkgo.It("should create snapshot objects correctly and delete according to its deletion policy", func() { ginkgo.By("checking the snapshot") // Get new copy of the snapshot vs, err = dc.Resource(SnapshotGVR).Namespace(vs.GetNamespace()).Get(context.TODO(), vs.GetName(), metav1.GetOptions{}) @@ -243,6 +227,21 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], vsc.GetName()) framework.ExpectEqual(volumeSnapshotRef["name"], vs.GetName()) framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace()) + + ginkgo.By("should delete the VolumeSnapshotContent according to its deletion policy") + err = DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + + switch pattern.SnapshotDeletionPolicy { + case testpatterns.DeleteSnapshot: + ginkgo.By("checking the SnapshotContent has been deleted") + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + case testpatterns.RetainSnapshot: + ginkgo.By("checking the SnapshotContent has not been deleted") + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), 1*time.Second /* poll */, 30*time.Second /* timeout */) + framework.ExpectError(err) + } }) ginkgo.It("should restore from snapshot with saved data after modifying source data", func() { var restoredPVC *v1.PersistentVolumeClaim From 854645d7c3b2d08686a00962b2253e8ed6eeb728 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Thu, 23 Jul 2020 15:59:32 +0000 Subject: [PATCH 04/11] Remove snapshot class from prepv test because it is not needed. Prepv snapshots only need to specify driver. --- test/e2e/storage/testsuites/base.go | 7 +++---- test/e2e/storage/testsuites/snapshottable.go | 7 +++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 079d07a9b2d..0a5c851b2b5 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -591,7 +591,7 @@ func getPreProvisionedSnapshot(snapshotContentName, ns, snapshotHandle string) * return snapshot } -func getPreProvisionedSnapshotContent(snapshotName, snapshotNamespace, snapshotHandle, deletionPolicy, csiDriverName, volumeSnapshotClassName string) *unstructured.Unstructured { +func getPreProvisionedSnapshotContent(snapshotName, snapshotNamespace, snapshotHandle, deletionPolicy, csiDriverName string) *unstructured.Unstructured { snapshotContent := &unstructured.Unstructured{ Object: map[string]interface{}{ "kind": "VolumeSnapshotContent", @@ -607,9 +607,8 @@ func getPreProvisionedSnapshotContent(snapshotName, snapshotNamespace, snapshotH "name": snapshotName, "namespace": snapshotNamespace, }, - "driver": csiDriverName, - "deletionPolicy": deletionPolicy, - "volumeSnapshotClassName": volumeSnapshotClassName, + "driver": csiDriverName, + "deletionPolicy": deletionPolicy, }, }, } diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index c6bb19c0f0f..6bc174ddf03 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -224,7 +224,10 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt // Check SnapshotContent properties ginkgo.By("checking the SnapshotContent") - framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], vsc.GetName()) + // PreprovisionedCreatedSnapshot do not need to set volume snapshot class name + if pattern.SnapshotType != testpatterns.PreprovisionedCreatedSnapshot { + framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], vsc.GetName()) + } framework.ExpectEqual(volumeSnapshotRef["name"], vs.GetName()) framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace()) @@ -451,7 +454,7 @@ func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConf framework.ExpectNoError(err) ginkgo.By("creating a snapshot content with the snapshot handle") - r.Vscontent = getPreProvisionedSnapshotContent(getPreProvisionedSnapshotName(snapshotHandle), pvcNamespace, snapshotHandle, pattern.SnapshotDeletionPolicy.String(), csiDriverName, r.Vsclass.GetName()) + r.Vscontent = getPreProvisionedSnapshotContent(getPreProvisionedSnapshotName(snapshotHandle), pvcNamespace, snapshotHandle, pattern.SnapshotDeletionPolicy.String(), csiDriverName) r.Vscontent, err = dc.Resource(SnapshotContentGVR).Create(context.TODO(), r.Vscontent, metav1.CreateOptions{}) framework.ExpectNoError(err) From a4498846b1dede1118bb15712abb38764bd22e3e Mon Sep 17 00:00:00 2001 From: Andi Li Date: Tue, 28 Jul 2020 17:17:19 +0000 Subject: [PATCH 05/11] Quick commit --- test/e2e/storage/testsuites/base.go | 12 +++--------- test/e2e/storage/testsuites/snapshottable.go | 8 ++++++-- test/e2e/storage/utils/utils.go | 9 +++++++++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 0a5c851b2b5..fbafa06d78d 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -18,7 +18,6 @@ package testsuites import ( "context" - "crypto/sha1" "flag" "fmt" "math" @@ -47,6 +46,7 @@ import ( e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" "k8s.io/kubernetes/test/e2e/storage/podlogs" "k8s.io/kubernetes/test/e2e/storage/testpatterns" + "k8s.io/kubernetes/test/e2e/storage/utils" ) var ( @@ -616,18 +616,12 @@ func getPreProvisionedSnapshotContent(snapshotName, snapshotNamespace, snapshotH return snapshotContent } -func genShortHash(s string) string { - h := sha1.New() - h.Write([]byte(s)) - bs := h.Sum(nil) - return fmt.Sprintf("%x", bs)[:7] -} func getPreProvisionedSnapshotContentName(snapshotHandle string) string { - return fmt.Sprintf("pre-provisioned-snapcontent-%s", genShortHash(snapshotHandle)) + return fmt.Sprintf("pre-provisioned-snapcontent-%s", utils.GenShortHash(snapshotHandle)) } func getPreProvisionedSnapshotName(snapshotHandle string) string { - return fmt.Sprintf("pre-provisioned-snapshot-%s", genShortHash(snapshotHandle)) + return fmt.Sprintf("pre-provisioned-snapshot-%s", utils.GenShortHash(snapshotHandle)) } // StartPodLogs begins capturing log output and events from current diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 6bc174ddf03..b21eb678d20 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -339,7 +339,7 @@ func DeleteAndWaitSnapshot(dc dynamic.Interface, ns string, snapshotName string, } ginkgo.By("checking the Snapshot has been deleted") - err = utils.WaitForNamespacedGVRDeletion(dc, SnapshotGVR, snapshotName, ns, poll, timeout) + err = utils.WaitForNamespacedGVRDeletion(dc, SnapshotGVR, ns, snapshotName, poll, timeout) return err } @@ -438,7 +438,11 @@ func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConf framework.Logf("Recording snapshot handle: %s", snapshotHandle) csiDriverName := r.Vsclass.Object["driver"].(string) - ginkgo.By("deleting the snapshot and snapshot content") // TODO: test what happens when I have two snapshot content that refer to the same content + // If the deletion policy is retain on vscontent: + // when vs is deleted vscontent will not be deleted + // when the vscontent is manually deleted then the underlying snapshot resource will not be deleted. + // We exploit this to create a snapshot resource from which we can create a preprovisioned snapshot + ginkgo.By("deleting the snapshot and snapshot content") err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Delete(context.TODO(), r.Vs.GetName(), metav1.DeleteOptions{}) framework.ExpectNoError(err) diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index 7ee13f4f9ac..46efa31d4e5 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -18,6 +18,7 @@ package utils import ( "context" + "crypto/sha1" "crypto/sha256" "encoding/base64" "fmt" @@ -785,3 +786,11 @@ func WaitUntil(poll, timeout time.Duration, checkDone func() bool) bool { framework.Logf("WaitUntil failed after reaching the timeout %v", timeout) return false } + +// GenShortHash returns the first 7 hex characters of the sha1 hash of string s +func GenShortHash(s string) string { + h := sha1.New() + h.Write([]byte(s)) + bs := h.Sum(nil) + return fmt.Sprintf("%x", bs)[:7] +} From 86af5a5ec3cd15678e3aa924c4b2ae26d4537e81 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Wed, 5 Aug 2020 22:12:12 +0000 Subject: [PATCH 06/11] Stack snapshot tests into one to reuse snapshot resource and reduce time taken on prow. --- test/e2e/storage/testsuites/snapshottable.go | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index b21eb678d20..78c390f4e1d 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -207,7 +207,7 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt vscontent = sr.Vscontent vsc = sr.Vsclass }) - ginkgo.It("should create snapshot objects correctly and delete according to its deletion policy", func() { + ginkgo.It("should check snapshot fields, check restore correctly works after modifying source data, check deletion", func() { ginkgo.By("checking the snapshot") // Get new copy of the snapshot vs, err = dc.Resource(SnapshotGVR).Namespace(vs.GetNamespace()).Get(context.TODO(), vs.GetName(), metav1.GetOptions{}) @@ -231,22 +231,7 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt framework.ExpectEqual(volumeSnapshotRef["name"], vs.GetName()) framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace()) - ginkgo.By("should delete the VolumeSnapshotContent according to its deletion policy") - err = DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err) - - switch pattern.SnapshotDeletionPolicy { - case testpatterns.DeleteSnapshot: - ginkgo.By("checking the SnapshotContent has been deleted") - err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err) - case testpatterns.RetainSnapshot: - ginkgo.By("checking the SnapshotContent has not been deleted") - err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), 1*time.Second /* poll */, 30*time.Second /* timeout */) - framework.ExpectError(err) - } - }) - ginkgo.It("should restore from snapshot with saved data after modifying source data", func() { + ginkgo.By("Modifying source data test") var restoredPVC *v1.PersistentVolumeClaim var restoredPod *v1.Pod modifiedMntTestData := fmt.Sprintf("modified data from %s namespace", pvc.GetNamespace()) @@ -293,6 +278,21 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt actualData, err := utils.PodExec(f, restoredPod, command) framework.ExpectNoError(err) framework.ExpectEqual(actualData, originalMntTestData) + + ginkgo.By("should delete the VolumeSnapshotContent according to its deletion policy") + err = DeleteAndWaitSnapshot(dc, vs.GetNamespace(), vs.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + + switch pattern.SnapshotDeletionPolicy { + case testpatterns.DeleteSnapshot: + ginkgo.By("checking the SnapshotContent has been deleted") + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err) + case testpatterns.RetainSnapshot: + ginkgo.By("checking the SnapshotContent has not been deleted") + err = utils.WaitForGVRDeletion(dc, SnapshotContentGVR, vscontent.GetName(), 1*time.Second /* poll */, 30*time.Second /* timeout */) + framework.ExpectError(err) + } }) }) }) From 7918f6905562ceae3725278ce907abdce1abde95 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Thu, 6 Aug 2020 02:31:08 +0000 Subject: [PATCH 07/11] Remove custom hash function, generate a random UUID for snapshot and snapcontent name instead --- test/e2e/storage/testsuites/BUILD | 2 ++ test/e2e/storage/testsuites/base.go | 18 +++++++++--------- test/e2e/storage/testsuites/snapshottable.go | 10 ++++++++-- test/e2e/storage/utils/utils.go | 9 --------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/test/e2e/storage/testsuites/BUILD b/test/e2e/storage/testsuites/BUILD index f27f44e2d5b..a2324b6cc9a 100644 --- a/test/e2e/storage/testsuites/BUILD +++ b/test/e2e/storage/testsuites/BUILD @@ -35,10 +35,12 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index fbafa06d78d..c36af57f4db 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" @@ -46,7 +47,6 @@ import ( e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" "k8s.io/kubernetes/test/e2e/storage/podlogs" "k8s.io/kubernetes/test/e2e/storage/testpatterns" - "k8s.io/kubernetes/test/e2e/storage/utils" ) var ( @@ -572,13 +572,13 @@ func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.U return snapshot } -func getPreProvisionedSnapshot(snapshotContentName, ns, snapshotHandle string) *unstructured.Unstructured { +func getPreProvisionedSnapshot(snapName, ns, snapshotContentName string) *unstructured.Unstructured { snapshot := &unstructured.Unstructured{ Object: map[string]interface{}{ "kind": "VolumeSnapshot", "apiVersion": snapshotAPIVersion, "metadata": map[string]interface{}{ - "name": getPreProvisionedSnapshotName(snapshotHandle), + "name": snapName, "namespace": ns, }, "spec": map[string]interface{}{ @@ -591,13 +591,13 @@ func getPreProvisionedSnapshot(snapshotContentName, ns, snapshotHandle string) * return snapshot } -func getPreProvisionedSnapshotContent(snapshotName, snapshotNamespace, snapshotHandle, deletionPolicy, csiDriverName string) *unstructured.Unstructured { +func getPreProvisionedSnapshotContent(snapcontentName, snapshotName, snapshotNamespace, snapshotHandle, deletionPolicy, csiDriverName string) *unstructured.Unstructured { snapshotContent := &unstructured.Unstructured{ Object: map[string]interface{}{ "kind": "VolumeSnapshotContent", "apiVersion": snapshotAPIVersion, "metadata": map[string]interface{}{ - "name": getPreProvisionedSnapshotContentName(snapshotHandle), + "name": snapcontentName, }, "spec": map[string]interface{}{ "source": map[string]interface{}{ @@ -616,12 +616,12 @@ func getPreProvisionedSnapshotContent(snapshotName, snapshotNamespace, snapshotH return snapshotContent } -func getPreProvisionedSnapshotContentName(snapshotHandle string) string { - return fmt.Sprintf("pre-provisioned-snapcontent-%s", utils.GenShortHash(snapshotHandle)) +func getPreProvisionedSnapshotContentName(uuid types.UID) string { + return fmt.Sprintf("pre-provisioned-snapcontent-%s", string(uuid)) } -func getPreProvisionedSnapshotName(snapshotHandle string) string { - return fmt.Sprintf("pre-provisioned-snapshot-%s", utils.GenShortHash(snapshotHandle)) +func getPreProvisionedSnapshotName(uuid types.UID) string { + return fmt.Sprintf("pre-provisioned-snapshot-%s", string(uuid)) } // StartPodLogs begins capturing log output and events from current diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 78c390f4e1d..148a8e22dd3 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -458,12 +459,17 @@ func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConf framework.ExpectNoError(err) ginkgo.By("creating a snapshot content with the snapshot handle") - r.Vscontent = getPreProvisionedSnapshotContent(getPreProvisionedSnapshotName(snapshotHandle), pvcNamespace, snapshotHandle, pattern.SnapshotDeletionPolicy.String(), csiDriverName) + uuid := uuid.NewUUID() + + snapName := getPreProvisionedSnapshotName(uuid) + snapcontentName := getPreProvisionedSnapshotContentName(uuid) + + r.Vscontent = getPreProvisionedSnapshotContent(snapcontentName, snapName, pvcNamespace, snapshotHandle, pattern.SnapshotDeletionPolicy.String(), csiDriverName) r.Vscontent, err = dc.Resource(SnapshotContentGVR).Create(context.TODO(), r.Vscontent, metav1.CreateOptions{}) framework.ExpectNoError(err) ginkgo.By("creating a snapshot with that snapshot content") - r.Vs = getPreProvisionedSnapshot(getPreProvisionedSnapshotContentName(snapshotHandle), pvcNamespace, snapshotHandle) + r.Vs = getPreProvisionedSnapshot(snapName, pvcNamespace, snapcontentName) r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{}) framework.ExpectNoError(err) diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index 46efa31d4e5..7ee13f4f9ac 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -18,7 +18,6 @@ package utils import ( "context" - "crypto/sha1" "crypto/sha256" "encoding/base64" "fmt" @@ -786,11 +785,3 @@ func WaitUntil(poll, timeout time.Duration, checkDone func() bool) bool { framework.Logf("WaitUntil failed after reaching the timeout %v", timeout) return false } - -// GenShortHash returns the first 7 hex characters of the sha1 hash of string s -func GenShortHash(s string) string { - h := sha1.New() - h.Write([]byte(s)) - bs := h.Sum(nil) - return fmt.Sprintf("%x", bs)[:7] -} From e748ec3de2bc2e62665dcdcd5d2e6523fbd4eb8a Mon Sep 17 00:00:00 2001 From: Andi Li Date: Thu, 6 Aug 2020 19:17:18 +0000 Subject: [PATCH 08/11] Move common portion of dynamic snapshot out of switch statement --- test/e2e/storage/testsuites/snapshottable.go | 56 +++++++------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 148a8e22dd3..3323a05997c 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -382,53 +382,35 @@ func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConf r.Vsclass, err = dc.Resource(SnapshotClassGVR).Get(context.TODO(), r.Vsclass.GetName(), metav1.GetOptions{}) framework.ExpectNoError(err) - switch pattern.SnapshotType { - case testpatterns.DynamicCreatedSnapshot: - ginkgo.By("creating a VolumeSnapshot") - // prepare a dynamically provisioned volume snapshot with certain data - r.Vs = getSnapshot(pvcName, pvcNamespace, r.Vsclass.GetName()) + ginkgo.By("creating a dynamic VolumeSnapshot") + // prepare a dynamically provisioned volume snapshot with certain data + r.Vs = getSnapshot(pvcName, pvcNamespace, r.Vsclass.GetName()) - r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{}) - framework.ExpectNoError(err) + r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{}) + framework.ExpectNoError(err) - err = WaitForSnapshotReady(dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout) - framework.ExpectNoError(err) + err = WaitForSnapshotReady(dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout) + framework.ExpectNoError(err) - r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(context.TODO(), r.Vs.GetName(), metav1.GetOptions{}) + r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(context.TODO(), r.Vs.GetName(), metav1.GetOptions{}) - snapshotStatus := r.Vs.Object["status"].(map[string]interface{}) - snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) - framework.Logf("received snapshotStatus %v", snapshotStatus) - framework.Logf("snapshotContentName %s", snapshotContentName) - framework.ExpectNoError(err) + snapshotStatus := r.Vs.Object["status"].(map[string]interface{}) + snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) + framework.Logf("received snapshotStatus %v", snapshotStatus) + framework.Logf("snapshotContentName %s", snapshotContentName) + framework.ExpectNoError(err) - r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) - framework.ExpectNoError(err) - case testpatterns.PreprovisionedCreatedSnapshot: + r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + if pattern.SnapshotType == testpatterns.PreprovisionedCreatedSnapshot { // prepare a pre-provisioned VolumeSnapshotContent with certain data // Because this could be run with an external CSI driver, we have no way // to pre-provision the snapshot as we normally would using their API. - // We instead dynamically take a snapshot and create another snapshot using - // the first snapshot's snapshot handle. - ginkgo.By("taking a dynamic snapshot") - r.Vs = getSnapshot(pvcName, pvcNamespace, r.Vsclass.GetName()) - r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Create(context.TODO(), r.Vs, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - err = WaitForSnapshotReady(dc, r.Vs.GetNamespace(), r.Vs.GetName(), framework.Poll, framework.SnapshotCreateTimeout) - framework.ExpectNoError(err) - - r.Vs, err = dc.Resource(SnapshotGVR).Namespace(r.Vs.GetNamespace()).Get(context.TODO(), r.Vs.GetName(), metav1.GetOptions{}) - - snapshotStatus := r.Vs.Object["status"].(map[string]interface{}) - snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) - framework.Logf("received snapshotStatus %v", snapshotStatus) - framework.Logf("snapshotContentName %s", snapshotContentName) - framework.ExpectNoError(err) + // We instead dynamically take a snapshot (above step), delete the old snapshot, + // and create another snapshot using the first snapshot's snapshot handle. ginkgo.By("updating the snapshot content deletion policy to retain") - r.Vscontent, err = dc.Resource(SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{}) - framework.ExpectNoError(err) r.Vscontent.Object["spec"].(map[string]interface{})["deletionPolicy"] = "Retain" r.Vscontent, err = dc.Resource(SnapshotContentGVR).Update(context.TODO(), r.Vscontent, metav1.UpdateOptions{}) From 5b4dffb6b70a938e71f66515673e6a5d2fb9d2d5 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Fri, 7 Aug 2020 21:31:23 +0000 Subject: [PATCH 09/11] Ran hack/update-bazel.sh but discarded changes to root build file --- test/e2e/storage/testsuites/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/storage/testsuites/BUILD b/test/e2e/storage/testsuites/BUILD index a2324b6cc9a..c7427df2c81 100644 --- a/test/e2e/storage/testsuites/BUILD +++ b/test/e2e/storage/testsuites/BUILD @@ -39,8 +39,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", From d915463451cb41c081440e59c2bc95eb151db5e2 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Thu, 27 Aug 2020 22:32:27 -0400 Subject: [PATCH 10/11] Combine switch case into one case --- test/e2e/storage/external/external.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/e2e/storage/external/external.go b/test/e2e/storage/external/external.go index ac0eb515a22..fbd7678f5c1 100644 --- a/test/e2e/storage/external/external.go +++ b/test/e2e/storage/external/external.go @@ -251,11 +251,7 @@ func (d *driverDefinition) SkipUnsupportedTest(pattern testpatterns.TestPattern) switch pattern.SnapshotType { case "": supported = true - case testpatterns.DynamicCreatedSnapshot: - if d.SnapshotClass.FromName || d.SnapshotClass.FromFile != "" || d.SnapshotClass.FromExistingClassName != "" { - supported = true - } - case testpatterns.PreprovisionedCreatedSnapshot: + case testpatterns.DynamicCreatedSnapshot, testpatterns.PreprovisionedCreatedSnapshot: if d.SnapshotClass.FromName || d.SnapshotClass.FromFile != "" || d.SnapshotClass.FromExistingClassName != "" { supported = true } From 673af97bcd7d5c7f6ebfc4b97dca4c8822875055 Mon Sep 17 00:00:00 2001 From: Andi Li Date: Fri, 28 Aug 2020 13:44:42 -0400 Subject: [PATCH 11/11] Fix testpattern check --- test/e2e/storage/testsuites/snapshottable.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 3323a05997c..e5dc3a63848 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -359,7 +359,7 @@ type SnapshotResource struct { // different test pattern snapshot provisioning and deletion policy func CreateSnapshotResource(sDriver SnapshottableTestDriver, config *PerTestConfig, pattern testpatterns.TestPattern, pvcName string, pvcNamespace string) *SnapshotResource { var err error - if pattern.SnapshotType != testpatterns.DynamicCreatedSnapshot || pattern.SnapshotType != testpatterns.PreprovisionedCreatedSnapshot { + if pattern.SnapshotType != testpatterns.DynamicCreatedSnapshot && pattern.SnapshotType != testpatterns.PreprovisionedCreatedSnapshot { err = fmt.Errorf("SnapshotType must be set to either DynamicCreatedSnapshot or PreprovisionedCreatedSnapshot") framework.ExpectNoError(err) }