From 5462d97e62297cc588bc8dda2976db1b9ec1cf98 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 13 Oct 2021 14:13:48 +0200 Subject: [PATCH] storage e2e: test snapshotting of generic ephemeral volumes Conceptually, snapshots have to be taken while the pod and thus the volume exist. Snapshotting has an issue where flushing of data is not guaranteed while the volume is still staged on the node, so the test relied on deleting the pod and checking for the volume to be unused. That part of the test cannot be done for ephmeral volumes. --- test/e2e/storage/framework/testpattern.go | 16 +++ test/e2e/storage/testsuites/provisioning.go | 19 ++- test/e2e/storage/testsuites/snapshottable.go | 132 +++++++++++++------ 3 files changed, 123 insertions(+), 44 deletions(-) diff --git a/test/e2e/storage/framework/testpattern.go b/test/e2e/storage/framework/testpattern.go index 3a61d1ba1c6..32421764d49 100644 --- a/test/e2e/storage/framework/testpattern.go +++ b/test/e2e/storage/framework/testpattern.go @@ -314,6 +314,14 @@ var ( SnapshotDeletionPolicy: DeleteSnapshot, VolType: DynamicPV, } + // EphemeralSnapshotDelete is TestPattern for snapshotting of a generic ephemeral volume + // where snapshots are deleted. + EphemeralSnapshotDelete = TestPattern{ + Name: "Ephemeral Snapshot (delete policy)", + SnapshotType: DynamicCreatedSnapshot, + SnapshotDeletionPolicy: DeleteSnapshot, + VolType: GenericEphemeralVolume, + } // DynamicSnapshotRetain is TestPattern for "Dynamic snapshot" DynamicSnapshotRetain = TestPattern{ Name: "Dynamic Snapshot (retain policy)", @@ -328,6 +336,14 @@ var ( SnapshotDeletionPolicy: RetainSnapshot, VolType: DynamicPV, } + // EphemeralSnapshotDelete is TestPattern for snapshotting of a generic ephemeral volume + // where snapshots are preserved. + EphemeralSnapshotRetain = TestPattern{ + Name: "Ephemeral Snapshot (retain policy)", + SnapshotType: DynamicCreatedSnapshot, + SnapshotDeletionPolicy: RetainSnapshot, + VolType: GenericEphemeralVolume, + } // Definitions for volume expansion case diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 03717e210b6..eab48a2092c 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -698,6 +698,16 @@ func RunInPodWithVolume(c clientset.Interface, t *framework.TimeoutContext, ns, // StartInPodWithVolume starts a command in a pod with given claim mounted to /mnt directory // The caller is responsible for checking the pod and deleting it. func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node e2epod.NodeSelection) *v1.Pod { + return StartInPodWithVolumeSource(c, v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: claimName, + }, + }, ns, podName, command, node) +} + +// StartInPodWithVolumeSource starts a command in a pod with given volume mounted to /mnt directory +// The caller is responsible for checking the pod and deleting it. +func StartInPodWithVolumeSource(c clientset.Interface, volSrc v1.VolumeSource, ns, podName, command string, node e2epod.NodeSelection) *v1.Pod { pod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -726,13 +736,8 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command RestartPolicy: v1.RestartPolicyNever, Volumes: []v1.Volume{ { - Name: "my-volume", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: claimName, - ReadOnly: false, - }, - }, + Name: "my-volume", + VolumeSource: volSrc, }, }, }, diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index ab8ed97ea00..866c970b7fa 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" + "k8s.io/component-helpers/storage/ephemeral" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" @@ -75,6 +76,8 @@ func InitSnapshottableTestSuite() storageframework.TestSuite { patterns := []storageframework.TestPattern{ storageframework.DynamicSnapshotDelete, storageframework.DynamicSnapshotRetain, + storageframework.EphemeralSnapshotDelete, + storageframework.EphemeralSnapshotRetain, storageframework.PreprovisionedSnapshotDelete, storageframework.PreprovisionedSnapshotRetain, } @@ -137,29 +140,61 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver, }) volumeResource = storageframework.CreateVolumeResource(dDriver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange) - pvc = volumeResource.Pvc - sc = volumeResource.Sc - claimSize = pvc.Spec.Resources.Requests.Storage().String() + pvcName := "" + pvcNamespace := f.Namespace.Name ginkgo.By("[init] starting a pod to use the claim") - originalMntTestData = fmt.Sprintf("hello from %s namespace", pvc.GetNamespace()) + originalMntTestData = fmt.Sprintf("hello from %s namespace", pvcNamespace) command := fmt.Sprintf("echo '%s' > %s", originalMntTestData, datapath) - pod := RunInPodWithVolume(cs, f.Timeouts, pvc.Namespace, pvc.Name, "pvc-snapshottable-tester", command, config.ClientNodeSelection) - - err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, pvc.Namespace, pvc.Name, framework.Poll, f.Timeouts.ClaimProvision) - framework.ExpectNoError(err) + pod := StartInPodWithVolumeSource(cs, *volumeResource.VolSource, f.Namespace.Name, "pvc-snapshottable-tester", command, config.ClientNodeSelection) + if pattern.VolType == storageframework.GenericEphemeralVolume { + // We can test snapshotting of generic ephemeral volumes. It's just a bit more complicated: + // - we need to start the pod + // - wait for the pod to stop + // - don't delete the pod because once it is marked for deletion, + // the PVC also gets marked and snapshotting no longer works + // (even when a finalizer prevents actual removal of the PVC) + // - skip data validation because data flushing wasn't guaranteed + // (see comments below) + cleanupSteps = append(cleanupSteps, func() { + e2epod.DeletePodWithWait(cs, pod) + }) + } else { + defer e2epod.DeletePodWithWait(cs, pod) + } + framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceTimeout(cs, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow)) + pod, err = cs.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "check pod after it terminated") // Get new copy of the claim ginkgo.By("[init] checking the claim") - pvc, err = cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{}) + if pattern.VolType == storageframework.GenericEphemeralVolume { + pvcName = ephemeral.VolumeClaimName(pod, &pod.Spec.Volumes[0]) + } else { + pvcName = volumeResource.Pvc.Name + } + + err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, pvcNamespace, pvcName, framework.Poll, f.Timeouts.ClaimProvision) framework.ExpectNoError(err) + pvc, err = cs.CoreV1().PersistentVolumeClaims(pvcNamespace).Get(context.TODO(), pvcName, metav1.GetOptions{}) + framework.ExpectNoError(err, "get PVC") + claimSize = pvc.Spec.Resources.Requests.Storage().String() + sc = volumeResource.Sc + // Get the bound PV ginkgo.By("[init] checking the PV") pv, err := cs.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{}) framework.ExpectNoError(err) + if pattern.VolType == storageframework.GenericEphemeralVolume { + return + } + + ginkgo.By("[init] deleting the pod") + StopPod(cs, pod) + // At this point we know that: // - a pod was created with a PV that's supposed to have data // @@ -260,6 +295,9 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver, snapshotContentSpec := vscontent.Object["spec"].(map[string]interface{}) volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{}) + var restoredPVC *v1.PersistentVolumeClaim + var restoredPod *v1.Pod + // Check SnapshotContent properties ginkgo.By("checking the SnapshotContent") // PreprovisionedCreatedSnapshot do not need to set volume snapshot class name @@ -269,15 +307,15 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver, framework.ExpectEqual(volumeSnapshotRef["name"], vs.GetName()) framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace()) - ginkgo.By("Modifying source data test") - var restoredPVC *v1.PersistentVolumeClaim - var restoredPod *v1.Pod - modifiedMntTestData := fmt.Sprintf("modified data from %s namespace", pvc.GetNamespace()) + if pattern.VolType != storageframework.GenericEphemeralVolume { + ginkgo.By("Modifying source data test") + modifiedMntTestData := fmt.Sprintf("modified data from %s namespace", pvc.GetNamespace()) - ginkgo.By("modifying the data in the source PVC") + ginkgo.By("modifying the data in the source PVC") - command := fmt.Sprintf("echo '%s' > %s", modifiedMntTestData, datapath) - RunInPodWithVolume(cs, f.Timeouts, pvc.Namespace, pvc.Name, "pvc-snapshottable-data-tester", command, config.ClientNodeSelection) + command := fmt.Sprintf("echo '%s' > %s", modifiedMntTestData, datapath) + RunInPodWithVolume(cs, f.Timeouts, pvc.Namespace, pvc.Name, "pvc-snapshottable-data-tester", command, config.ClientNodeSelection) + } ginkgo.By("creating a pvc from the snapshot") restoredPVC = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ @@ -293,27 +331,45 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver, Name: vs.GetName(), } - restoredPVC, err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Create(context.TODO(), restoredPVC, metav1.CreateOptions{}) - framework.ExpectNoError(err) - cleanupSteps = append(cleanupSteps, func() { - framework.Logf("deleting claim %q/%q", restoredPVC.Namespace, restoredPVC.Name) - // typically this claim has already been deleted - err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Delete(context.TODO(), restoredPVC.Name, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - framework.Failf("Error deleting claim %q. Error: %v", restoredPVC.Name, err) + if pattern.VolType != storageframework.GenericEphemeralVolume { + restoredPVC, err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Create(context.TODO(), restoredPVC, metav1.CreateOptions{}) + framework.ExpectNoError(err) + cleanupSteps = append(cleanupSteps, func() { + framework.Logf("deleting claim %q/%q", restoredPVC.Namespace, restoredPVC.Name) + // typically this claim has already been deleted + err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Delete(context.TODO(), restoredPVC.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Failf("Error deleting claim %q. Error: %v", restoredPVC.Name, err) + } + }) + } + + ginkgo.By("starting a pod to use the snapshot") + volSrc := v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: restoredPVC.Name, + }, + } + if pattern.VolType == storageframework.GenericEphemeralVolume { + volSrc = v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: restoredPVC.Spec, + }, + }, } - }) + } - ginkgo.By("starting a pod to use the claim") - - restoredPod = StartInPodWithVolume(cs, restoredPVC.Namespace, restoredPVC.Name, "restored-pvc-tester", "sleep 300", config.ClientNodeSelection) + restoredPod = StartInPodWithVolumeSource(cs, volSrc, restoredPVC.Namespace, "restored-pvc-tester", "sleep 300", config.ClientNodeSelection) cleanupSteps = append(cleanupSteps, func() { StopPod(cs, restoredPod) }) framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(cs, restoredPod.Name, restoredPod.Namespace, f.Timeouts.PodStartSlow)) - commands := e2evolume.GenerateReadFileCmd(datapath) - _, err = framework.LookForStringInPodExec(restoredPod.Namespace, restoredPod.Name, commands, originalMntTestData, time.Minute) - framework.ExpectNoError(err) + if pattern.VolType != storageframework.GenericEphemeralVolume { + commands := e2evolume.GenerateReadFileCmd(datapath) + _, err = framework.LookForStringInPodExec(restoredPod.Namespace, restoredPod.Name, commands, originalMntTestData, time.Minute) + framework.ExpectNoError(err) + } ginkgo.By("should delete the VolumeSnapshotContent according to its deletion policy") @@ -322,12 +378,14 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver, // Snapshot deletion and some are opposite. err = storageutils.DeleteSnapshotWithoutWaiting(dc, vs.GetNamespace(), vs.GetName()) framework.ExpectNoError(err) - framework.Logf("deleting restored pod %q/%q", restoredPod.Namespace, restoredPod.Name) - err = cs.CoreV1().Pods(restoredPod.Namespace).Delete(context.TODO(), restoredPod.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err) - framework.Logf("deleting restored PVC %q/%q", restoredPVC.Namespace, restoredPVC.Name) - err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Delete(context.TODO(), restoredPVC.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err) + if pattern.VolType != storageframework.GenericEphemeralVolume { + framework.Logf("deleting restored pod %q/%q", restoredPod.Namespace, restoredPod.Name) + err = cs.CoreV1().Pods(restoredPod.Namespace).Delete(context.TODO(), restoredPod.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + framework.Logf("deleting restored PVC %q/%q", restoredPVC.Namespace, restoredPVC.Name) + err = cs.CoreV1().PersistentVolumeClaims(restoredPVC.Namespace).Delete(context.TODO(), restoredPVC.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + } // Wait for the Snapshot to be actually deleted from API server err = storageutils.WaitForNamespacedGVRDeletion(dc, storageutils.SnapshotGVR, vs.GetNamespace(), vs.GetNamespace(), framework.Poll, f.Timeouts.SnapshotDelete)