Merge pull request #105659 from pohly/generic-ephemeral-volume-test

generic ephemeral volume E2E tests

test/e2e/storage/framework/testpattern.go

@@ -314,6 +314,14 @@ var (
 		SnapshotDeletionPolicy: DeleteSnapshot,
 		VolType:                DynamicPV,
 	}
+	// EphemeralSnapshotDelete is TestPattern for snapshotting of a generic ephemeral volume
+	// where snapshots are deleted.
+	EphemeralSnapshotDelete = TestPattern{
+		Name:                   "Ephemeral Snapshot (delete policy)",
+		SnapshotType:           DynamicCreatedSnapshot,
+		SnapshotDeletionPolicy: DeleteSnapshot,
+		VolType:                GenericEphemeralVolume,
+	}
 	// DynamicSnapshotRetain is TestPattern for "Dynamic snapshot"
 	DynamicSnapshotRetain = TestPattern{
 		Name:                   "Dynamic Snapshot (retain policy)",
@@ -328,6 +336,14 @@ var (
 		SnapshotDeletionPolicy: RetainSnapshot,
 		VolType:                DynamicPV,
 	}
+	// EphemeralSnapshotDelete is TestPattern for snapshotting of a generic ephemeral volume
+	// where snapshots are preserved.
+	EphemeralSnapshotRetain = TestPattern{
+		Name:                   "Ephemeral Snapshot (retain policy)",
+		SnapshotType:           DynamicCreatedSnapshot,
+		SnapshotDeletionPolicy: RetainSnapshot,
+		VolType:                GenericEphemeralVolume,
+	}
 
 	// Definitions for volume expansion case
 

test/e2e/storage/testsuites/provisioning.go

@@ -698,6 +698,16 @@ func RunInPodWithVolume(c clientset.Interface, t *framework.TimeoutContext, ns,
 // StartInPodWithVolume starts a command in a pod with given claim mounted to /mnt directory
 // The caller is responsible for checking the pod and deleting it.
 func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node e2epod.NodeSelection) *v1.Pod {
+	return StartInPodWithVolumeSource(c, v1.VolumeSource{
+		PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+			ClaimName: claimName,
+		},
+	}, ns, podName, command, node)
+}
+
+// StartInPodWithVolumeSource starts a command in a pod with given volume mounted to /mnt directory
+// The caller is responsible for checking the pod and deleting it.
+func StartInPodWithVolumeSource(c clientset.Interface, volSrc v1.VolumeSource, ns, podName, command string, node e2epod.NodeSelection) *v1.Pod {
 	pod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Pod",
@@ -726,13 +736,8 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command
 			RestartPolicy: v1.RestartPolicyNever,
 			Volumes: []v1.Volume{
 				{
-					Name: "my-volume",
+					Name:         "my-volume",
-					VolumeSource: v1.VolumeSource{
+					VolumeSource: volSrc,
-						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
-							ClaimName: claimName,
-							ReadOnly:  false,
-						},
-					},
 				},
 			},
 		},

test/e2e/storage/testsuites/snapshottable.go

@@ -29,9 +29,9 @@ import (
 	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/component-helpers/storage/ephemeral"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
@@ -75,6 +75,8 @@ func InitSnapshottableTestSuite() storageframework.TestSuite {
 	patterns := []storageframework.TestPattern{
 		storageframework.DynamicSnapshotDelete,
 		storageframework.DynamicSnapshotRetain,
+		storageframework.EphemeralSnapshotDelete,
+		storageframework.EphemeralSnapshotRetain,
 		storageframework.PreprovisionedSnapshotDelete,
 		storageframework.PreprovisionedSnapshotRetain,
 	}
@@ -116,6 +118,8 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver,
 			dc                  dynamic.Interface
 			pvc                 *v1.PersistentVolumeClaim
 			sc                  *storagev1.StorageClass
+			volumeResource      *storageframework.VolumeResource
+			pod                 *v1.Pod
 			claimSize           string
 			originalMntTestData string
 		)
@@ -131,78 +135,21 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver,
 			config, driverCleanup = driver.PrepareTest(f)
 			cleanupSteps = append(cleanupSteps, driverCleanup)
 
-			var volumeResource *storageframework.VolumeResource
 			cleanupSteps = append(cleanupSteps, func() {
 				framework.ExpectNoError(volumeResource.CleanupResource())
 			})
 			volumeResource = storageframework.CreateVolumeResource(dDriver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange)
 
-			pvc = volumeResource.Pvc
-			sc = volumeResource.Sc
-			claimSize = pvc.Spec.Resources.Requests.Storage().String()
-
 			ginkgo.By("[init] starting a pod to use the claim")
-			originalMntTestData = fmt.Sprintf("hello from %s namespace", pvc.GetNamespace())
+			originalMntTestData = fmt.Sprintf("hello from %s namespace", f.Namespace.Name)
 			command := fmt.Sprintf("echo '%s' > %s", originalMntTestData, datapath)
 
-			pod := RunInPodWithVolume(cs, f.Timeouts, pvc.Namespace, pvc.Name, "pvc-snapshottable-tester", command, config.ClientNodeSelection)
+			pod = StartInPodWithVolumeSource(cs, *volumeResource.VolSource, f.Namespace.Name, "pvc-snapshottable-tester", command, config.ClientNodeSelection)
-
+			cleanupSteps = append(cleanupSteps, func() {
-			err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, pvc.Namespace, pvc.Name, framework.Poll, f.Timeouts.ClaimProvision)
+				e2epod.DeletePodWithWait(cs, pod)
-			framework.ExpectNoError(err)
-
-			// Get new copy of the claim
-			ginkgo.By("[init] checking the claim")
-			pvc, err = cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{})
-			framework.ExpectNoError(err)
-
-			// Get the bound PV
-			ginkgo.By("[init] checking the PV")
-			pv, err := cs.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
-			framework.ExpectNoError(err)
-
-			// At this point we know that:
-			// - a pod was created with a PV that's supposed to have data
-			//
-			// However there's a caching issue that @jinxu97 explained and it's related with the pod & volume
-			// lifecycle, to understand it we first analyze what the volumemanager does:
-			// - when a pod is delete the volumemanager will try to cleanup the volume mounts
-			//   - NodeUnpublishVolume: unbinds the bind mount from the container
-			//     - Linux: the bind mount is removed, which does not flush any cache
-			//     - Windows: we delete a symlink, data's not flushed yet to disk
-			//   - NodeUnstageVolume: unmount the global mount
-			//     - Linux: disk is unmounted and all caches flushed.
-			//     - Windows: data is flushed to disk and the disk is detached
-			//
-			// Pod deletion might not guarantee a data flush to disk, however NodeUnstageVolume adds the logic
-			// to flush the data to disk (see #81690 for details). We need to wait for NodeUnstageVolume, as
-			// NodeUnpublishVolume only removes the bind mount, which doesn't force the caches to flush.
-			// It's possible to create empty snapshots if we don't wait (see #101279 for details).
-			//
-			// In the following code by checking if the PV is not in the node.Status.VolumesInUse field we
-			// ensure that the volume is not used by the node anymore (an indicator that NodeUnstageVolume has
-			// already finished)
-			nodeName := pod.Spec.NodeName
-			gomega.Expect(nodeName).NotTo(gomega.BeEmpty(), "pod.Spec.NodeName must not be empty")
-
-			// Snapshot tests are only executed for CSI drivers. When CSI drivers
-			// are attached to the node they use VolumeHandle instead of the pv.Name.
-			volumeName := pv.Spec.PersistentVolumeSource.CSI.VolumeHandle
-
-			ginkgo.By(fmt.Sprintf("[init] waiting until the node=%s is not using the volume=%s", nodeName, volumeName))
-			success := storageutils.WaitUntil(framework.Poll, f.Timeouts.PVDelete, func() bool {
-				node, err := cs.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
-				framework.ExpectNoError(err)
-				volumesInUse := node.Status.VolumesInUse
-				framework.Logf("current volumes in use: %+v", volumesInUse)
-				for i := 0; i < len(volumesInUse); i++ {
-					if strings.HasSuffix(string(volumesInUse[i]), volumeName) {
-						return false
-					}
-				}
-				return true
 			})
-			framework.ExpectEqual(success, true)
 
+			// At this point a pod is running with a PVC. How to proceed depends on which test is running.
 		}
 
 		cleanup := func() {
@@ -220,32 +167,57 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver,
 			}
 
 		}
-		ginkgo.BeforeEach(func() {
-			init()
-		})
 		ginkgo.AfterEach(func() {
 			cleanup()
 		})
 
 		ginkgo.Context("", func() {
-			var (
+			ginkgo.It("should check snapshot fields, check restore correctly works, check deletion (ephemeral)", func() {
-				vs        *unstructured.Unstructured
+				if pattern.VolType != storageframework.GenericEphemeralVolume {
-				vscontent *unstructured.Unstructured
+					e2eskipper.Skipf("volume type %q is not ephemeral", pattern.VolType)
-				vsc       *unstructured.Unstructured
+				}
-			)
+				init()
 
-			ginkgo.BeforeEach(func() {
+				// We can test snapshotting of generic
-				var sr *storageframework.SnapshotResource
+				// ephemeral volumes by creating the snapshot
+				// while the pod is running (online). We cannot do it after pod deletion,
+				// because then the PVC also gets marked and snapshotting no longer works
+				// (even when a finalizer prevents actual removal of the PVC).
+				//
+				// Because data consistency cannot be
+				// guaranteed, this flavor of the test doesn't
+				// check the content of the snapshot.
+
+				framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceTimeout(cs, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow))
+				pod, err = cs.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
+				framework.ExpectNoError(err, "check pod after it terminated")
+
+				// Get new copy of the claim
+				ginkgo.By("[init] checking the claim")
+				pvcName := ephemeral.VolumeClaimName(pod, &pod.Spec.Volumes[0])
+				pvcNamespace := pod.Namespace
+
+				parameters := map[string]string{}
+				sr := storageframework.CreateSnapshotResource(sDriver, config, pattern, pvcName, pvcNamespace, f.Timeouts, parameters)
 				cleanupSteps = append(cleanupSteps, func() {
 					framework.ExpectNoError(sr.CleanupResource(f.Timeouts))
 				})
-				parameters := map[string]string{}
+				vs := sr.Vs
-				sr = storageframework.CreateSnapshotResource(sDriver, config, pattern, pvc.GetName(), pvc.GetNamespace(), f.Timeouts, parameters)
+				vsc := sr.Vsclass
-				vs = sr.Vs
+
-				vscontent = sr.Vscontent
+				err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, pvcNamespace, pvcName, framework.Poll, f.Timeouts.ClaimProvision)
-				vsc = sr.Vsclass
+				framework.ExpectNoError(err)
-			})
+
-			ginkgo.It("should check snapshot fields, check restore correctly works after modifying source data, check deletion", func() {
+				pvc, err = cs.CoreV1().PersistentVolumeClaims(pvcNamespace).Get(context.TODO(), pvcName, metav1.GetOptions{})
+				framework.ExpectNoError(err, "get PVC")
+				claimSize = pvc.Spec.Resources.Requests.Storage().String()
+				sc = volumeResource.Sc
+
+				// Get the bound PV
+				ginkgo.By("[init] checking the PV")
+				_, err := cs.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
+				framework.ExpectNoError(err)
+
 				// Get new copy of the snapshot
 				ginkgo.By("checking the snapshot")
 				vs, err = dc.Resource(storageutils.SnapshotGVR).Namespace(vs.GetNamespace()).Get(context.TODO(), vs.GetName(), metav1.GetOptions{})
@@ -254,12 +226,183 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver,
 				// Get the bound snapshotContent
 				snapshotStatus := vs.Object["status"].(map[string]interface{})
 				snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string)
-				vscontent, err = dc.Resource(storageutils.SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{})
+				vscontent, err := dc.Resource(storageutils.SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{})
 				framework.ExpectNoError(err)
 
 				snapshotContentSpec := vscontent.Object["spec"].(map[string]interface{})
 				volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{})
 
+				var restoredPVC *v1.PersistentVolumeClaim
+				var restoredPod *v1.Pod
+
+				// Check SnapshotContent properties
+				ginkgo.By("checking the SnapshotContent")
+				// PreprovisionedCreatedSnapshot do not need to set volume snapshot class name
+				if pattern.SnapshotType != storageframework.PreprovisionedCreatedSnapshot {
+					framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], vsc.GetName())
+				}
+				framework.ExpectEqual(volumeSnapshotRef["name"], vs.GetName())
+				framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace())
+
+				ginkgo.By("creating a pvc from the snapshot")
+				restoredPVC = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
+					ClaimSize:        claimSize,
+					StorageClassName: &(sc.Name),
+				}, config.Framework.Namespace.Name)
+
+				group := "snapshot.storage.k8s.io"
+
+				restoredPVC.Spec.DataSource = &v1.TypedLocalObjectReference{
+					APIGroup: &group,
+					Kind:     "VolumeSnapshot",
+					Name:     vs.GetName(),
+				}
+
+				ginkgo.By("starting a pod to use the snapshot")
+				volSrc := v1.VolumeSource{
+					Ephemeral: &v1.EphemeralVolumeSource{
+						VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{
+							Spec: restoredPVC.Spec,
+						},
+					},
+				}
+
+				restoredPod = StartInPodWithVolumeSource(cs, volSrc, restoredPVC.Namespace, "restored-pvc-tester", "sleep 300", config.ClientNodeSelection)
+				cleanupSteps = append(cleanupSteps, func() {
+					StopPod(cs, restoredPod)
+				})
+				framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(cs, restoredPod.Name, restoredPod.Namespace, f.Timeouts.PodStartSlow))
+				if pattern.VolType != storageframework.GenericEphemeralVolume {
+					commands := e2evolume.GenerateReadFileCmd(datapath)
+					_, err = framework.LookForStringInPodExec(restoredPod.Namespace, restoredPod.Name, commands, originalMntTestData, time.Minute)
+					framework.ExpectNoError(err)
+				}
+
+				ginkgo.By("should delete the VolumeSnapshotContent according to its deletion policy")
+
+				// Delete both Snapshot and restored Pod/PVC at the same time because different storage systems
+				// have different ordering of deletion. Some may require delete the restored PVC first before
+				// Snapshot deletion and some are opposite.
+				err = storageutils.DeleteSnapshotWithoutWaiting(dc, vs.GetNamespace(), vs.GetName())
+				framework.ExpectNoError(err)
+
+				// Wait for the Snapshot to be actually deleted from API server
+				err = storageutils.WaitForNamespacedGVRDeletion(dc, storageutils.SnapshotGVR, vs.GetNamespace(), vs.GetNamespace(), framework.Poll, f.Timeouts.SnapshotDelete)
+				framework.ExpectNoError(err)
+
+				switch pattern.SnapshotDeletionPolicy {
+				case storageframework.DeleteSnapshot:
+					ginkgo.By("checking the SnapshotContent has been deleted")
+					err = utils.WaitForGVRDeletion(dc, storageutils.SnapshotContentGVR, vscontent.GetName(), framework.Poll, f.Timeouts.SnapshotDelete)
+					framework.ExpectNoError(err)
+				case storageframework.RetainSnapshot:
+					ginkgo.By("checking the SnapshotContent has not been deleted")
+					err = utils.WaitForGVRDeletion(dc, storageutils.SnapshotContentGVR, vscontent.GetName(), 1*time.Second /* poll */, 30*time.Second /* timeout */)
+					framework.ExpectError(err)
+				}
+			})
+
+			ginkgo.It("should check snapshot fields, check restore correctly works after modifying source data, check deletion (persistent)", func() {
+				if pattern.VolType == storageframework.GenericEphemeralVolume {
+					e2eskipper.Skipf("volume type %q is ephemeral", pattern.VolType)
+				}
+				init()
+
+				framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceTimeout(cs, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow))
+				pod, err = cs.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
+				framework.ExpectNoError(err, "check pod after it terminated")
+
+				// Get new copy of the claim
+				ginkgo.By("[init] checking the claim")
+				pvcName := volumeResource.Pvc.Name
+				pvcNamespace := volumeResource.Pvc.Namespace
+
+				parameters := map[string]string{}
+				sr := storageframework.CreateSnapshotResource(sDriver, config, pattern, pvcName, pvcNamespace, f.Timeouts, parameters)
+				cleanupSteps = append(cleanupSteps, func() {
+					framework.ExpectNoError(sr.CleanupResource(f.Timeouts))
+				})
+				vs := sr.Vs
+				vsc := sr.Vsclass
+
+				err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, cs, pvcNamespace, pvcName, framework.Poll, f.Timeouts.ClaimProvision)
+				framework.ExpectNoError(err)
+
+				pvc, err = cs.CoreV1().PersistentVolumeClaims(pvcNamespace).Get(context.TODO(), pvcName, metav1.GetOptions{})
+				framework.ExpectNoError(err, "get PVC")
+				claimSize = pvc.Spec.Resources.Requests.Storage().String()
+				sc = volumeResource.Sc
+
+				// Get the bound PV
+				ginkgo.By("[init] checking the PV")
+				pv, err := cs.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
+				framework.ExpectNoError(err)
+
+				ginkgo.By("[init] deleting the pod")
+				StopPod(cs, pod)
+
+				// At this point we know that:
+				// - a pod was created with a PV that's supposed to have data
+				//
+				// However there's a caching issue that @jinxu97 explained and it's related with the pod & volume
+				// lifecycle, to understand it we first analyze what the volumemanager does:
+				// - when a pod is delete the volumemanager will try to cleanup the volume mounts
+				//   - NodeUnpublishVolume: unbinds the bind mount from the container
+				//     - Linux: the bind mount is removed, which does not flush any cache
+				//     - Windows: we delete a symlink, data's not flushed yet to disk
+				//   - NodeUnstageVolume: unmount the global mount
+				//     - Linux: disk is unmounted and all caches flushed.
+				//     - Windows: data is flushed to disk and the disk is detached
+				//
+				// Pod deletion might not guarantee a data flush to disk, however NodeUnstageVolume adds the logic
+				// to flush the data to disk (see #81690 for details). We need to wait for NodeUnstageVolume, as
+				// NodeUnpublishVolume only removes the bind mount, which doesn't force the caches to flush.
+				// It's possible to create empty snapshots if we don't wait (see #101279 for details).
+				//
+				// In the following code by checking if the PV is not in the node.Status.VolumesInUse field we
+				// ensure that the volume is not used by the node anymore (an indicator that NodeUnstageVolume has
+				// already finished)
+				nodeName := pod.Spec.NodeName
+				gomega.Expect(nodeName).NotTo(gomega.BeEmpty(), "pod.Spec.NodeName must not be empty")
+
+				// Snapshot tests are only executed for CSI drivers. When CSI drivers
+				// are attached to the node they use VolumeHandle instead of the pv.Name.
+				volumeName := pv.Spec.PersistentVolumeSource.CSI.VolumeHandle
+
+				ginkgo.By(fmt.Sprintf("[init] waiting until the node=%s is not using the volume=%s", nodeName, volumeName))
+				success := storageutils.WaitUntil(framework.Poll, f.Timeouts.PVDelete, func() bool {
+					node, err := cs.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+					framework.ExpectNoError(err)
+					volumesInUse := node.Status.VolumesInUse
+					framework.Logf("current volumes in use: %+v", volumesInUse)
+					for i := 0; i < len(volumesInUse); i++ {
+						if strings.HasSuffix(string(volumesInUse[i]), volumeName) {
+							return false
+						}
+					}
+					return true
+				})
+				if !success {
+					framework.Failf("timed out waiting for node=%s to not use the volume=%s", nodeName, volumeName)
+				}
+
+				// Get new copy of the snapshot
+				ginkgo.By("checking the snapshot")
+				vs, err = dc.Resource(storageutils.SnapshotGVR).Namespace(vs.GetNamespace()).Get(context.TODO(), vs.GetName(), metav1.GetOptions{})
+				framework.ExpectNoError(err)
+
+				// Get the bound snapshotContent
+				snapshotStatus := vs.Object["status"].(map[string]interface{})
+				snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string)
+				vscontent, err := dc.Resource(storageutils.SnapshotContentGVR).Get(context.TODO(), snapshotContentName, metav1.GetOptions{})
+				framework.ExpectNoError(err)
+
+				snapshotContentSpec := vscontent.Object["spec"].(map[string]interface{})
+				volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{})
+
+				var restoredPVC *v1.PersistentVolumeClaim
+				var restoredPod *v1.Pod
+
 				// Check SnapshotContent properties
 				ginkgo.By("checking the SnapshotContent")
 				// PreprovisionedCreatedSnapshot do not need to set volume snapshot class name
@@ -270,8 +413,6 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver,
 				framework.ExpectEqual(volumeSnapshotRef["namespace"], vs.GetNamespace())
 
 				ginkgo.By("Modifying source data test")
-				var restoredPVC *v1.PersistentVolumeClaim
-				var restoredPod *v1.Pod
 				modifiedMntTestData := fmt.Sprintf("modified data from %s namespace", pvc.GetNamespace())
 
 				ginkgo.By("modifying the data in the source PVC")
@@ -304,16 +445,17 @@ func (s *snapshottableTestSuite) DefineTests(driver storageframework.TestDriver,
 					}
 				})
 
-				ginkgo.By("starting a pod to use the claim")
+				ginkgo.By("starting a pod to use the snapshot")
-
 				restoredPod = StartInPodWithVolume(cs, restoredPVC.Namespace, restoredPVC.Name, "restored-pvc-tester", "sleep 300", config.ClientNodeSelection)
 				cleanupSteps = append(cleanupSteps, func() {
 					StopPod(cs, restoredPod)
 				})
 				framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(cs, restoredPod.Name, restoredPod.Namespace, f.Timeouts.PodStartSlow))
-				commands := e2evolume.GenerateReadFileCmd(datapath)
+				if pattern.VolType != storageframework.GenericEphemeralVolume {
-				_, err = framework.LookForStringInPodExec(restoredPod.Namespace, restoredPod.Name, commands, originalMntTestData, time.Minute)
+					commands := e2evolume.GenerateReadFileCmd(datapath)
-				framework.ExpectNoError(err)
+					_, err = framework.LookForStringInPodExec(restoredPod.Namespace, restoredPod.Name, commands, originalMntTestData, time.Minute)
+					framework.ExpectNoError(err)
+				}
 
 				ginkgo.By("should delete the VolumeSnapshotContent according to its deletion policy")
 

test/e2e/storage/testsuites/volumelimits.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/component-helpers/storage/ephemeral"
 	migrationplugins "k8s.io/csi-translation-lib/plugins" // volume plugin names are exported nicely there
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -74,6 +75,7 @@ func InitCustomVolumeLimitsTestSuite(patterns []storageframework.TestPattern) st
 func InitVolumeLimitsTestSuite() storageframework.TestSuite {
 	patterns := []storageframework.TestPattern{
 		storageframework.FsVolModeDynamicPV,
+		storageframework.DefaultFsGenericEphemeralVolume,
 	}
 	return InitCustomVolumeLimitsTestSuite(patterns)
 }
@@ -95,14 +97,14 @@ func (t *volumeLimitsTestSuite) DefineTests(driver storageframework.TestDriver,
 		// VolumeResource contains pv, pvc, sc, etc. of the first pod created
 		resource *storageframework.VolumeResource
 
-		// All created PVCs, incl. the one in resource
+		// All created PVCs
-		pvcs []*v1.PersistentVolumeClaim
+		pvcNames []string
+
+		// All created Pods
+		podNames []string
 
 		// All created PVs, incl. the one in resource
 		pvNames sets.String
-
-		runningPod       *v1.Pod
-		unschedulablePod *v1.Pod
 	}
 	var (
 		l local
@@ -164,57 +166,64 @@ func (t *volumeLimitsTestSuite) DefineTests(driver storageframework.TestDriver,
 			framework.ExpectNoError(err, "while cleaning up resource")
 		}()
 		defer func() {
-			cleanupTest(l.cs, l.ns.Name, l.runningPod.Name, l.unschedulablePod.Name, l.pvcs, l.pvNames, testSlowMultiplier*f.Timeouts.PVDelete)
+			cleanupTest(l.cs, l.ns.Name, l.podNames, l.pvcNames, l.pvNames, testSlowMultiplier*f.Timeouts.PVDelete)
 		}()
 
-		// Create <limit> PVCs for one gigantic pod.
-		ginkgo.By(fmt.Sprintf("Creating %d PVC(s)", limit))
-
-		for i := 0; i < limit; i++ {
-			pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
-				ClaimSize:        claimSize,
-				StorageClassName: &l.resource.Sc.Name,
-			}, l.ns.Name)
-			pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
-			framework.ExpectNoError(err)
-			l.pvcs = append(l.pvcs, pvc)
-		}
-
-		ginkgo.By("Creating pod to use all PVC(s)")
 		selection := e2epod.NodeSelection{Name: nodeName}
-		podConfig := e2epod.Config{
+
-			NS:            l.ns.Name,
+		if pattern.VolType == storageframework.GenericEphemeralVolume {
-			PVCs:          l.pvcs,
+			// Create <limit> Pods.
-			SeLinuxLabel:  e2epv.SELinuxLabel,
+			ginkgo.By(fmt.Sprintf("Creating %d Pod(s) with one volume each", limit))
-			NodeSelection: selection,
+			for i := 0; i < limit; i++ {
+				pod := StartInPodWithVolumeSource(l.cs, *l.resource.VolSource, l.ns.Name, "volume-limits", "sleep 1000000", selection)
+				l.podNames = append(l.podNames, pod.Name)
+				l.pvcNames = append(l.pvcNames, ephemeral.VolumeClaimName(pod, &pod.Spec.Volumes[0]))
+			}
+		} else {
+			// Create <limit> PVCs for one gigantic pod.
+			var pvcs []*v1.PersistentVolumeClaim
+			ginkgo.By(fmt.Sprintf("Creating %d PVC(s)", limit))
+			for i := 0; i < limit; i++ {
+				pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
+					ClaimSize:        claimSize,
+					StorageClassName: &l.resource.Sc.Name,
+				}, l.ns.Name)
+				pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(context.TODO(), pvc, metav1.CreateOptions{})
+				framework.ExpectNoError(err)
+				l.pvcNames = append(l.pvcNames, pvc.Name)
+				pvcs = append(pvcs, pvc)
+			}
+
+			ginkgo.By("Creating pod to use all PVC(s)")
+			podConfig := e2epod.Config{
+				NS:            l.ns.Name,
+				PVCs:          pvcs,
+				SeLinuxLabel:  e2epv.SELinuxLabel,
+				NodeSelection: selection,
+			}
+			pod, err := e2epod.MakeSecPod(&podConfig)
+			framework.ExpectNoError(err)
+			pod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{})
+			framework.ExpectNoError(err)
+			l.podNames = append(l.podNames, pod.Name)
 		}
-		pod, err := e2epod.MakeSecPod(&podConfig)
-		framework.ExpectNoError(err)
-		l.runningPod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{})
-		framework.ExpectNoError(err)
 
 		ginkgo.By("Waiting for all PVCs to get Bound")
-		l.pvNames, err = waitForAllPVCsBound(l.cs, testSlowMultiplier*f.Timeouts.PVBound, l.pvcs)
+		l.pvNames, err = waitForAllPVCsBound(l.cs, testSlowMultiplier*f.Timeouts.PVBound, l.ns.Name, l.pvcNames)
 		framework.ExpectNoError(err)
 
-		ginkgo.By("Waiting for the pod Running")
+		ginkgo.By("Waiting for the pod(s) running")
-		err = e2epod.WaitTimeoutForPodRunningInNamespace(l.cs, l.runningPod.Name, l.ns.Name, testSlowMultiplier*f.Timeouts.PodStart)
+		for _, podName := range l.podNames {
-		framework.ExpectNoError(err)
+			err = e2epod.WaitTimeoutForPodRunningInNamespace(l.cs, podName, l.ns.Name, testSlowMultiplier*f.Timeouts.PodStart)
+			framework.ExpectNoError(err)
+		}
 
 		ginkgo.By("Creating an extra pod with one volume to exceed the limit")
-		podConfig = e2epod.Config{
+		pod := StartInPodWithVolumeSource(l.cs, *l.resource.VolSource, l.ns.Name, "volume-limits-exceeded", "sleep 10000", selection)
-			NS:            l.ns.Name,
+		l.podNames = append(l.podNames, pod.Name)
-			PVCs:          []*v1.PersistentVolumeClaim{l.resource.Pvc},
-			SeLinuxLabel:  e2epv.SELinuxLabel,
-			NodeSelection: selection,
-		}
-		pod, err = e2epod.MakeSecPod(&podConfig)
-		framework.ExpectNoError(err)
-		l.unschedulablePod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{})
-		framework.ExpectNoError(err, "Failed to create an extra pod with one volume to exceed the limit")
 
 		ginkgo.By("Waiting for the pod to get unschedulable with the right message")
-		err = e2epod.WaitForPodCondition(l.cs, l.ns.Name, l.unschedulablePod.Name, "Unschedulable", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) {
+		err = e2epod.WaitForPodCondition(l.cs, l.ns.Name, pod.Name, "Unschedulable", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) {
 			if pod.Status.Phase == v1.PodPending {
 				reg, err := regexp.Compile(`max.+volume.+count`)
 				if err != nil {
@@ -270,24 +279,18 @@ func (t *volumeLimitsTestSuite) DefineTests(driver storageframework.TestDriver,
 	})
 }
 
-func cleanupTest(cs clientset.Interface, ns string, runningPodName, unschedulablePodName string, pvcs []*v1.PersistentVolumeClaim, pvNames sets.String, timeout time.Duration) error {
+func cleanupTest(cs clientset.Interface, ns string, podNames, pvcNames []string, pvNames sets.String, timeout time.Duration) error {
 	var cleanupErrors []string
-	if runningPodName != "" {
+	for _, podName := range podNames {
-		err := cs.CoreV1().Pods(ns).Delete(context.TODO(), runningPodName, metav1.DeleteOptions{})
+		err := cs.CoreV1().Pods(ns).Delete(context.TODO(), podName, metav1.DeleteOptions{})
 		if err != nil {
-			cleanupErrors = append(cleanupErrors, fmt.Sprintf("failed to delete pod %s: %s", runningPodName, err))
+			cleanupErrors = append(cleanupErrors, fmt.Sprintf("failed to delete pod %s: %s", podName, err))
 		}
 	}
-	if unschedulablePodName != "" {
+	for _, pvcName := range pvcNames {
-		err := cs.CoreV1().Pods(ns).Delete(context.TODO(), unschedulablePodName, metav1.DeleteOptions{})
+		err := cs.CoreV1().PersistentVolumeClaims(ns).Delete(context.TODO(), pvcName, metav1.DeleteOptions{})
-		if err != nil {
+		if !apierrors.IsNotFound(err) {
-			cleanupErrors = append(cleanupErrors, fmt.Sprintf("failed to delete pod %s: %s", unschedulablePodName, err))
+			cleanupErrors = append(cleanupErrors, fmt.Sprintf("failed to delete PVC %s: %s", pvcName, err))
-		}
-	}
-	for _, pvc := range pvcs {
-		err := cs.CoreV1().PersistentVolumeClaims(ns).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{})
-		if err != nil {
-			cleanupErrors = append(cleanupErrors, fmt.Sprintf("failed to delete PVC %s: %s", pvc.Name, err))
 		}
 	}
 	// Wait for the PVs to be deleted. It includes also pod and PVC deletion because of PVC protection.
@@ -323,12 +326,12 @@ func cleanupTest(cs clientset.Interface, ns string, runningPodName, unschedulabl
 }
 
 // waitForAllPVCsBound waits until the given PVCs are all bound. It then returns the bound PVC names as a set.
-func waitForAllPVCsBound(cs clientset.Interface, timeout time.Duration, pvcs []*v1.PersistentVolumeClaim) (sets.String, error) {
+func waitForAllPVCsBound(cs clientset.Interface, timeout time.Duration, ns string, pvcNames []string) (sets.String, error) {
 	pvNames := sets.NewString()
 	err := wait.Poll(5*time.Second, timeout, func() (bool, error) {
 		unbound := 0
-		for _, pvc := range pvcs {
+		for _, pvcName := range pvcNames {
-			pvc, err := cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{})
+			pvc, err := cs.CoreV1().PersistentVolumeClaims(ns).Get(context.TODO(), pvcName, metav1.GetOptions{})
 			if err != nil {
 				return false, err
 			}
@@ -339,7 +342,7 @@ func waitForAllPVCsBound(cs clientset.Interface, timeout time.Duration, pvcs []*
 			}
 		}
 		if unbound > 0 {
-			framework.Logf("%d/%d of PVCs are Bound", pvNames.Len(), len(pvcs))
+			framework.Logf("%d/%d of PVCs are Bound", pvNames.Len(), len(pvcNames))
 			return false, nil
 		}
 		return true, nil

test/e2e/storage/volume_metrics.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/component-base/metrics/testutil"
+	"k8s.io/component-helpers/storage/ephemeral"
 	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
@@ -211,14 +212,15 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		verifyMetricCount(storageOpMetrics, updatedStorageMetrics, "volume_provision", true)
 	}
 
-	filesystemMode := func(ephemeral bool) {
+	filesystemMode := func(isEphemeral bool) {
-		if !ephemeral {
+		pvcName := pvc.Name
+		if !isEphemeral {
 			pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
 			framework.ExpectNoError(err)
 			framework.ExpectNotEqual(pvc, nil)
 		}
 
-		pod := makePod(ns, pvc, ephemeral)
+		pod := makePod(ns, pvc, isEphemeral)
 		pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
 		framework.ExpectNoError(err)
 
@@ -228,6 +230,11 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		pod, err = c.CoreV1().Pods(ns).Get(context.TODO(), pod.Name, metav1.GetOptions{})
 		framework.ExpectNoError(err)
 
+		if isEphemeral {
+			pvcName = ephemeral.VolumeClaimName(pod, &pod.Spec.Volumes[0])
+		}
+		pvcNamespace := pod.Namespace
+
 		// Verify volume stat metrics were collected for the referenced PVC
 		volumeStatKeys := []string{
 			kubeletmetrics.VolumeStatsUsedBytesKey,
@@ -251,31 +258,31 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 				framework.Logf("Error fetching kubelet metrics")
 				return false, err
 			}
-			if !findVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics) {
+			if !findVolumeStatMetric(kubeletKeyName, pvcNamespace, pvcName, kubeMetrics) {
 				return false, nil
 			}
 			return true, nil
 		})
-		framework.ExpectNoError(waitErr, "Unable to find metric %s for PVC %s/%s", kubeletKeyName, pvc.Namespace, pvc.Name)
+		framework.ExpectNoError(waitErr, "Unable to find metric %s for PVC %s/%s", kubeletKeyName, pvcNamespace, pvcName)
 
 		for _, key := range volumeStatKeys {
 			kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key)
-			found := findVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics)
+			found := findVolumeStatMetric(kubeletKeyName, pvcNamespace, pvcName, kubeMetrics)
-			framework.ExpectEqual(found, true, "PVC %s, Namespace %s not found for %s", pvc.Name, pvc.Namespace, kubeletKeyName)
+			framework.ExpectEqual(found, true, "PVC %s, Namespace %s not found for %s", pvcName, pvcNamespace, kubeletKeyName)
 		}
 
 		framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
 		framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
 	}
 
-	blockmode := func(ephemeral bool) {
+	blockmode := func(isEphemeral bool) {
-		if !ephemeral {
+		if !isEphemeral {
 			pvcBlock, err = c.CoreV1().PersistentVolumeClaims(pvcBlock.Namespace).Create(context.TODO(), pvcBlock, metav1.CreateOptions{})
 			framework.ExpectNoError(err)
 			framework.ExpectNotEqual(pvcBlock, nil)
 		}
 
-		pod := makePod(ns, pvcBlock, ephemeral)
+		pod := makePod(ns, pvcBlock, isEphemeral)
 		pod.Spec.Containers[0].VolumeDevices = []v1.VolumeDevice{{
 			Name:       pod.Spec.Volumes[0].Name,
 			DevicePath: "/mnt/" + pvcBlock.Name,
@@ -326,14 +333,14 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
 	}
 
-	totalTime := func(ephemeral bool) {
+	totalTime := func(isEphemeral bool) {
-		if !ephemeral {
+		if !isEphemeral {
 			pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
 			framework.ExpectNoError(err)
 			framework.ExpectNotEqual(pvc, nil)
 		}
 
-		pod := makePod(ns, pvc, ephemeral)
+		pod := makePod(ns, pvc, isEphemeral)
 		pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
 		framework.ExpectNoError(err)
 
@@ -357,14 +364,14 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
 	}
 
-	volumeManager := func(ephemeral bool) {
+	volumeManager := func(isEphemeral bool) {
-		if !ephemeral {
+		if !isEphemeral {
 			pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
 			framework.ExpectNoError(err)
 			framework.ExpectNotEqual(pvc, nil)
 		}
 
-		pod := makePod(ns, pvc, ephemeral)
+		pod := makePod(ns, pvc, isEphemeral)
 		pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
 		framework.ExpectNoError(err)
 
@@ -387,14 +394,14 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
 	}
 
-	adController := func(ephemeral bool) {
+	adController := func(isEphemeral bool) {
-		if !ephemeral {
+		if !isEphemeral {
 			pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
 			framework.ExpectNoError(err)
 			framework.ExpectNotEqual(pvc, nil)
 		}
 
-		pod := makePod(ns, pvc, ephemeral)
+		pod := makePod(ns, pvc, isEphemeral)
 
 		// Get metrics
 		controllerMetrics, err := metricsGrabber.GrabFromControllerManager()
@@ -447,27 +454,27 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
 	}
 
-	testAll := func(ephemeral bool) {
+	testAll := func(isEphemeral bool) {
 		ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() {
-			provisioning(ephemeral)
+			provisioning(isEphemeral)
 		})
 		ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() {
-			provisioningError(ephemeral)
+			provisioningError(isEphemeral)
 		})
 		ginkgo.It("should create volume metrics with the correct FilesystemMode PVC ref", func() {
-			filesystemMode(ephemeral)
+			filesystemMode(isEphemeral)
 		})
 		ginkgo.It("should create volume metrics with the correct BlockMode PVC ref", func() {
-			blockmode(ephemeral)
+			blockmode(isEphemeral)
 		})
 		ginkgo.It("should create metrics for total time taken in volume operations in P/V Controller", func() {
-			totalTime(ephemeral)
+			totalTime(isEphemeral)
 		})
 		ginkgo.It("should create volume metrics in Volume Manager", func() {
-			volumeManager(ephemeral)
+			volumeManager(isEphemeral)
 		})
 		ginkgo.It("should create metrics for total number of volumes in A/D Controller", func() {
-			adController(ephemeral)
+			adController(isEphemeral)
 		})
 	}
 
@@ -873,10 +880,10 @@ func waitForADControllerStatesMetrics(metricsGrabber *e2emetrics.Grabber, metric
 
 // makePod creates a pod which either references the PVC or creates it via a
 // generic ephemeral volume claim template.
-func makePod(ns string, pvc *v1.PersistentVolumeClaim, ephemeral bool) *v1.Pod {
+func makePod(ns string, pvc *v1.PersistentVolumeClaim, isEphemeral bool) *v1.Pod {
 	claims := []*v1.PersistentVolumeClaim{pvc}
 	pod := e2epod.MakePod(ns, nil, claims, false, "")
-	if ephemeral {
+	if isEphemeral {
 		volSrc := pod.Spec.Volumes[0]
 		volSrc.PersistentVolumeClaim = nil
 		volSrc.Ephemeral = &v1.EphemeralVolumeSource{