CSI: allow drivers that can handle persistent and ephemeral volumes

The conceptual change is that the mode in which a volume gets handled
is derived from it's spec, not from the ability of the driver. In
practice, that is already how the code worked because it didn't
actually look at CSIDriver.Spec.Mode at all.

Therefore the code change itself is mostly just renaming "driver mode"
to "volume mode". In some places (CanDeviceMount, CanAttach) the
feature check that was used elsewhere seemed to be missing. Now their
code path for ephemeral volumes are also only entered if that feature
is enabled.

The sanity check whether a CSI driver is being used correctly still
needs to be implemented.

Related-to: https://github.com/kubernetes/kubernetes/issues/79624

pkg/volume/csi/csi_mounter.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 
 	"k8s.io/klog"
 
@@ -44,14 +45,14 @@ var (
 		driverName,
 		nodeName,
 		attachmentID,
-		driverMode string
+		csiVolumeMode string
 	}{
 		"specVolID",
 		"volumeHandle",
 		"driverName",
 		"nodeName",
 		"attachmentID",
-		"driverMode",
+		"csiVolumeMode",
 	}
 )
 
@@ -60,7 +61,7 @@ type csiMountMgr struct {
 	k8s            kubernetes.Interface
 	plugin         *csiPlugin
 	driverName     csiDriverName
-	driverMode     driverMode
+	csiVolumeMode  csiVolumeMode
 	volumeID       string
 	specVolumeID   string
 	readOnly       bool
@@ -146,8 +147,8 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error
 		if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) {
 			return fmt.Errorf("CSIInlineVolume feature required")
 		}
-		if c.driverMode != ephemeralDriverMode {
+		if c.csiVolumeMode != ephemeralVolumeMode {
-			return fmt.Errorf("unexpected driver mode: %s", c.driverMode)
+			return fmt.Errorf("unexpected volume mode: %s", c.csiVolumeMode)
 		}
 		if volSrc.FSType != nil {
 			fsType = *volSrc.FSType
@@ -161,8 +162,8 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error
 			secretRef = &api.SecretReference{Name: secretName, Namespace: ns}
 		}
 	case pvSrc != nil:
-		if c.driverMode != persistentDriverMode {
+		if c.csiVolumeMode != persistentVolumeMode {
-			return fmt.Errorf("unexpected driver mode: %s", c.driverMode)
+			return fmt.Errorf("unexpected driver mode: %s", c.csiVolumeMode)
 		}
 
 		fsType = pvSrc.FSType
@@ -324,6 +325,10 @@ func (c *csiMountMgr) podAttributes() (map[string]string, error) {
 		"csi.storage.k8s.io/pod.uid":             string(c.pod.UID),
 		"csi.storage.k8s.io/serviceAccount.name": c.pod.Spec.ServiceAccountName,
 	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) {
+		attrs["csi.storage.k8s.io/ephemeral"] = strconv.FormatBool(c.csiVolumeMode == ephemeralVolumeMode)
+	}
+
 	klog.V(4).Infof(log("CSIDriver %q requires pod information", c.driverName))
 	return attrs, nil
 }

pkg/volume/csi/csi_mounter_test.go

@@ -99,6 +99,7 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) {
 		driver                string
 		volumeContext         map[string]string
 		expectedVolumeContext map[string]string
+		csiInlineVolume       bool
 	}{
 		{
 			name:                  "no pod info",
@@ -136,6 +137,13 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) {
 			volumeContext:         map[string]string{"foo": "bar"},
 			expectedVolumeContext: map[string]string{"foo": "bar", "csi.storage.k8s.io/pod.uid": "test-pod", "csi.storage.k8s.io/serviceAccount.name": "test-service-account", "csi.storage.k8s.io/pod.name": "test-pod", "csi.storage.k8s.io/pod.namespace": "test-ns"},
 		},
+		{
+			name:                  "CSIInlineVolume pod info",
+			driver:                "info",
+			volumeContext:         nil,
+			expectedVolumeContext: map[string]string{"csi.storage.k8s.io/pod.uid": "test-pod", "csi.storage.k8s.io/serviceAccount.name": "test-service-account", "csi.storage.k8s.io/pod.name": "test-pod", "csi.storage.k8s.io/pod.namespace": "test-ns", "csi.storage.k8s.io/ephemeral": "false"},
+			csiInlineVolume:       true,
+		},
 	}
 
 	noPodMountInfo := false
@@ -143,6 +151,9 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			klog.Infof("Starting test %s", test.name)
+			if test.csiInlineVolume {
+				defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)()
+			}
 			fakeClient := fakeclient.NewSimpleClientset(
 				getTestCSIDriver("no-info", &noPodMountInfo, nil),
 				getTestCSIDriver("info", &currentPodInfoMount, nil),
@@ -267,7 +278,7 @@ func TestMounterSetUpSimple(t *testing.T) {
 	testCases := []struct {
 		name       string
 		podUID     types.UID
-		mode       driverMode
+		mode       csiVolumeMode
 		fsType     string
 		options    []string
 		spec       func(string, []string) *volume.Spec
@@ -276,7 +287,7 @@ func TestMounterSetUpSimple(t *testing.T) {
 		{
 			name:       "setup with vol source",
 			podUID:     types.UID(fmt.Sprintf("%08X", rand.Uint64())),
-			mode:       ephemeralDriverMode,
+			mode:       ephemeralVolumeMode,
 			fsType:     "ext4",
 			shouldFail: true,
 			spec: func(fsType string, options []string) *volume.Spec {
@@ -288,7 +299,7 @@ func TestMounterSetUpSimple(t *testing.T) {
 		{
 			name:   "setup with persistent source",
 			podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
-			mode:   persistentDriverMode,
+			mode:   persistentVolumeMode,
 			fsType: "zfs",
 			spec: func(fsType string, options []string) *volume.Spec {
 				pvSrc := makeTestPV("pv1", 20, testDriver, "vol1")
@@ -300,7 +311,7 @@ func TestMounterSetUpSimple(t *testing.T) {
 		{
 			name:   "setup with persistent source without unspecified fstype and options",
 			podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
-			mode:   persistentDriverMode,
+			mode:   persistentVolumeMode,
 			spec: func(fsType string, options []string) *volume.Spec {
 				return volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol2"), false)
 			},
@@ -334,8 +345,8 @@ func TestMounterSetUpSimple(t *testing.T) {
 			csiMounter := mounter.(*csiMountMgr)
 			csiMounter.csiClient = setupClient(t, true)
 
-			if csiMounter.driverMode != persistentDriverMode {
+			if csiMounter.csiVolumeMode != persistentVolumeMode {
-				t.Fatal("unexpected driver mode: ", csiMounter.driverMode)
+				t.Fatal("unexpected volume mode: ", csiMounter.csiVolumeMode)
 			}
 
 			attachID := getAttachmentName(csiMounter.volumeID, string(csiMounter.driverName), string(plug.host.GetNodeName()))
@@ -393,7 +404,7 @@ func TestMounterSetUpWithInline(t *testing.T) {
 	testCases := []struct {
 		name       string
 		podUID     types.UID
-		mode       driverMode
+		mode       csiVolumeMode
 		fsType     string
 		options    []string
 		spec       func(string, []string) *volume.Spec
@@ -402,7 +413,7 @@ func TestMounterSetUpWithInline(t *testing.T) {
 		{
 			name:   "setup with vol source",
 			podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
-			mode:   ephemeralDriverMode,
+			mode:   ephemeralVolumeMode,
 			fsType: "ext4",
 			spec: func(fsType string, options []string) *volume.Spec {
 				volSrc := makeTestVol("pv1", testDriver)
@@ -413,7 +424,7 @@ func TestMounterSetUpWithInline(t *testing.T) {
 		{
 			name:   "setup with persistent source",
 			podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
-			mode:   persistentDriverMode,
+			mode:   persistentVolumeMode,
 			fsType: "zfs",
 			spec: func(fsType string, options []string) *volume.Spec {
 				pvSrc := makeTestPV("pv1", 20, testDriver, "vol1")
@@ -425,7 +436,7 @@ func TestMounterSetUpWithInline(t *testing.T) {
 		{
 			name:   "setup with persistent source without unspecified fstype and options",
 			podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())),
-			mode:   persistentDriverMode,
+			mode:   persistentVolumeMode,
 			spec: func(fsType string, options []string) *volume.Spec {
 				return volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol2"), false)
 			},
@@ -459,15 +470,15 @@ func TestMounterSetUpWithInline(t *testing.T) {
 			csiMounter := mounter.(*csiMountMgr)
 			csiMounter.csiClient = setupClient(t, true)
 
-			if csiMounter.driverMode != tc.mode {
+			if csiMounter.csiVolumeMode != tc.mode {
-				t.Fatal("unexpected driver mode: ", csiMounter.driverMode)
+				t.Fatal("unexpected volume mode: ", csiMounter.csiVolumeMode)
 			}
 
-			if csiMounter.driverMode == ephemeralDriverMode && csiMounter.volumeID != makeVolumeHandle(string(tc.podUID), csiMounter.specVolumeID) {
+			if csiMounter.csiVolumeMode == ephemeralVolumeMode && csiMounter.volumeID != makeVolumeHandle(string(tc.podUID), csiMounter.specVolumeID) {
 				t.Fatal("unexpected generated volumeHandle:", csiMounter.volumeID)
 			}
 
-			if csiMounter.driverMode == persistentDriverMode {
+			if csiMounter.csiVolumeMode == persistentVolumeMode {
 				attachID := getAttachmentName(csiMounter.volumeID, string(csiMounter.driverName), string(plug.host.GetNodeName()))
 				attachment := makeTestAttachment(attachID, "test-node", csiMounter.spec.Name())
 				_, err = csiMounter.k8s.StorageV1().VolumeAttachments().Create(attachment)
@@ -492,10 +503,10 @@ func TestMounterSetUpWithInline(t *testing.T) {
 			}
 
 			// validate stagingTargetPath
-			if tc.mode == ephemeralDriverMode && vol.DeviceMountPath != "" {
+			if tc.mode == ephemeralVolumeMode && vol.DeviceMountPath != "" {
 				t.Errorf("unexpected devicePathTarget sent to driver: %s", vol.DeviceMountPath)
 			}
-			if tc.mode == persistentDriverMode {
+			if tc.mode == persistentVolumeMode {
 				devicePath, err := makeDeviceMountPath(plug, csiMounter.spec)
 				if err != nil {
 					t.Fatal(err)

pkg/volume/csi/csi_plugin.go

@@ -77,6 +77,12 @@ type driverMode string
 
 const persistentDriverMode driverMode = "persistent"
 const ephemeralDriverMode driverMode = "ephemeral"
+const combinedDriverMode driverMode = "persistent+ephemeral"
+
+type csiVolumeMode string
+
+const persistentVolumeMode csiVolumeMode = "persistent"
+const ephemeralVolumeMode csiVolumeMode = "ephemeral"
 
 // ProbeVolumePlugins returns implemented plugins
 func ProbeVolumePlugins() []volume.VolumePlugin {
@@ -381,11 +387,16 @@ func (p *csiPlugin) NewMounter(
 		return nil, fmt.Errorf("volume source not found in volume.Spec")
 	}
 
-	driverMode, err := p.getDriverMode(spec)
+	csiVolumeMode, err := p.getCSIVolumeMode(spec)
 	if err != nil {
 		return nil, err
 	}
 
+	// TODO(pohly): check CSIDriver.Spec.Mode to ensure that the CSI driver
+	// supports the current csiVolumeMode.
+	// In alpha it is assumed that drivers are used correctly without
+	// the additional sanity check.
+
 	k8s := p.host.GetKubeClient()
 	if k8s == nil {
 		klog.Error(log("failed to get a kubernetes client"))
@@ -405,7 +416,7 @@ func (p *csiPlugin) NewMounter(
 		pod:           pod,
 		podUID:        pod.UID,
 		driverName:    csiDriverName(driverName),
-		driverMode:   driverMode,
+		csiVolumeMode: csiVolumeMode,
 		volumeID:      volumeHandle,
 		specVolumeID:  spec.Name(),
 		readOnly:      readOnly,
@@ -432,7 +443,7 @@ func (p *csiPlugin) NewMounter(
 		volDataKey.volHandle:     volumeHandle,
 		volDataKey.driverName:    driverName,
 		volDataKey.nodeName:      node,
-		volDataKey.driverMode: string(driverMode),
+		volDataKey.csiVolumeMode: string(csiVolumeMode),
 	}
 
 	attachID := getAttachmentName(volumeHandle, driverName, node)
@@ -497,16 +508,13 @@ func (p *csiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.S
 	var spec *volume.Spec
 	inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume)
 
-	// If inlineEnabled is true and mode is ephemeralDriverMode,
+	// If inlineEnabled is true and mode is ephemeralVolumeMode,
 	// use constructVolSourceSpec to construct volume source spec.
-	// If inlineEnabled is false or mode is persistentDriverMode,
+	// If inlineEnabled is false or mode is persistentVolumeMode,
 	// use constructPVSourceSpec to construct volume construct pv source spec.
-	if inlineEnabled {
+	if inlineEnabled && csiVolumeMode(volData[volDataKey.csiVolumeMode]) == ephemeralVolumeMode {
-		if driverMode(volData[volDataKey.driverMode]) == ephemeralDriverMode {
 		spec = p.constructVolSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName])
 		return spec, nil
-
-		}
 	}
 	spec = p.constructPVSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName], volData[volDataKey.volHandle])
 
@@ -577,15 +585,18 @@ func (p *csiPlugin) NewDetacher() (volume.Detacher, error) {
 }
 
 func (p *csiPlugin) CanAttach(spec *volume.Spec) (bool, error) {
-	driverMode, err := p.getDriverMode(spec)
+	inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume)
+	if inlineEnabled {
+		csiVolumeMode, err := p.getCSIVolumeMode(spec)
 		if err != nil {
 			return false, err
 		}
 
-	if driverMode == ephemeralDriverMode {
+		if csiVolumeMode == ephemeralVolumeMode {
 			klog.V(5).Info(log("plugin.CanAttach = false, ephemeral mode detected for spec %v", spec.Name()))
 			return false, nil
 		}
+	}
 
 	pvSrc, err := getCSISourceFromSpec(spec)
 	if err != nil {
@@ -604,16 +615,23 @@ func (p *csiPlugin) CanAttach(spec *volume.Spec) (bool, error) {
 
 // CanDeviceMount returns true if the spec supports device mount
 func (p *csiPlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
-	driverMode, err := p.getDriverMode(spec)
+	inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume)
+	if !inlineEnabled {
+		// No need to check anything, we assume it is a persistent volume.
+		return true, nil
+	}
+
+	csiVolumeMode, err := p.getCSIVolumeMode(spec)
 	if err != nil {
 		return false, err
 	}
 
-	if driverMode == ephemeralDriverMode {
+	if csiVolumeMode == ephemeralVolumeMode {
 		klog.V(5).Info(log("plugin.CanDeviceMount skipped ephemeral mode detected for spec %v", spec.Name()))
 		return false, nil
 	}
 
+	// Persistent volumes support device mount.
 	return true, nil
 }
 
@@ -785,13 +803,11 @@ func (p *csiPlugin) skipAttach(driver string) (bool, error) {
 	return false, nil
 }
 
-// getDriverMode returns the driver mode for the specified spec: {persistent|ephemeral}.
+// getCSIVolumeMode returns the mode for the specified spec: {persistent|ephemeral}.
 // 1) If mode cannot be determined, it will default to "persistent".
 // 2) If Mode cannot be resolved to either {persistent | ephemeral}, an error is returned
 // See https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md
-func (p *csiPlugin) getDriverMode(spec *volume.Spec) (driverMode, error) {
+func (p *csiPlugin) getCSIVolumeMode(spec *volume.Spec) (csiVolumeMode, error) {
-	// TODO (vladimirvivien) ultimately, mode will be retrieved from CSIDriver.Spec.Mode.
-	// However, in alpha version, mode is determined by the volume source:
 	// 1) if volume.Spec.Volume.CSI != nil -> mode is ephemeral
 	// 2) if volume.Spec.PersistentVolume.Spec.CSI != nil -> persistent
 	volSrc, _, err := getSourceFromSpec(spec)
@@ -800,9 +816,9 @@ func (p *csiPlugin) getDriverMode(spec *volume.Spec) (driverMode, error) {
 	}
 
 	if volSrc != nil && utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) {
-		return ephemeralDriverMode, nil
+		return ephemeralVolumeMode, nil
 	}
-	return persistentDriverMode, nil
+	return persistentVolumeMode, nil
 }
 
 func (p *csiPlugin) getPublishContext(client clientset.Interface, handle, driver, nodeName string) (map[string]string, error) {

pkg/volume/csi/csi_plugin_test.go

@@ -524,7 +524,7 @@ func TestPluginNewMounter(t *testing.T) {
 		spec          *volume.Spec
 		podUID        types.UID
 		namespace     string
-		driverMode driverMode
+		csiVolumeMode csiVolumeMode
 		shouldFail    bool
 	}{
 		{
@@ -532,14 +532,14 @@ func TestPluginNewMounter(t *testing.T) {
 			spec:          volume.NewSpecFromPersistentVolume(makeTestPV("test-pv1", 20, testDriver, testVol), true),
 			podUID:        types.UID(fmt.Sprintf("%08X", rand.Uint64())),
 			namespace:     "test-ns1",
-			driverMode: persistentDriverMode,
+			csiVolumeMode: persistentVolumeMode,
 		},
 		{
 			name:          "mounter from volume source",
 			spec:          volume.NewSpecFromVolume(makeTestVol("test-vol1", testDriver)),
 			podUID:        types.UID(fmt.Sprintf("%08X", rand.Uint64())),
 			namespace:     "test-ns2",
-			driverMode: ephemeralDriverMode,
+			csiVolumeMode: ephemeralVolumeMode,
 			shouldFail:    true, // csi inline not enabled
 		},
 		{
@@ -590,8 +590,8 @@ func TestPluginNewMounter(t *testing.T) {
 			if csiClient == nil {
 				t.Error("mounter csiClient is nil")
 			}
-			if csiMounter.driverMode != test.driverMode {
+			if csiMounter.csiVolumeMode != test.csiVolumeMode {
-				t.Error("unexpected driver mode:", csiMounter.driverMode)
+				t.Error("unexpected driver mode:", csiMounter.csiVolumeMode)
 			}
 
 			// ensure data file is created
@@ -620,8 +620,8 @@ func TestPluginNewMounter(t *testing.T) {
 			if data[volDataKey.nodeName] != string(csiMounter.plugin.host.GetNodeName()) {
 				t.Error("volume data file unexpected nodeName:", data[volDataKey.nodeName])
 			}
-			if data[volDataKey.driverMode] != string(test.driverMode) {
+			if data[volDataKey.csiVolumeMode] != string(test.csiVolumeMode) {
-				t.Error("volume data file unexpected driverMode:", data[volDataKey.driverMode])
+				t.Error("volume data file unexpected csiVolumeMode:", data[volDataKey.csiVolumeMode])
 			}
 		})
 	}
@@ -635,7 +635,7 @@ func TestPluginNewMounterWithInline(t *testing.T) {
 		spec          *volume.Spec
 		podUID        types.UID
 		namespace     string
-		driverMode driverMode
+		csiVolumeMode csiVolumeMode
 		shouldFail    bool
 	}{
 		{
@@ -656,14 +656,14 @@ func TestPluginNewMounterWithInline(t *testing.T) {
 			spec:          volume.NewSpecFromPersistentVolume(makeTestPV("test-pv1", 20, testDriver, testVol), true),
 			podUID:        types.UID(fmt.Sprintf("%08X", rand.Uint64())),
 			namespace:     "test-ns1",
-			driverMode: persistentDriverMode,
+			csiVolumeMode: persistentVolumeMode,
 		},
 		{
 			name:          "mounter with volume source",
 			spec:          volume.NewSpecFromVolume(makeTestVol("test-vol1", testDriver)),
 			podUID:        types.UID(fmt.Sprintf("%08X", rand.Uint64())),
 			namespace:     "test-ns2",
-			driverMode: ephemeralDriverMode,
+			csiVolumeMode: ephemeralVolumeMode,
 		},
 	}
 
@@ -709,8 +709,8 @@ func TestPluginNewMounterWithInline(t *testing.T) {
 			if csiClient == nil {
 				t.Error("mounter csiClient is nil")
 			}
-			if csiMounter.driverMode != test.driverMode {
+			if csiMounter.csiVolumeMode != test.csiVolumeMode {
-				t.Error("unexpected driver mode:", csiMounter.driverMode)
+				t.Error("unexpected driver mode:", csiMounter.csiVolumeMode)
 			}
 
 			// ensure data file is created
@@ -739,8 +739,8 @@ func TestPluginNewMounterWithInline(t *testing.T) {
 			if data[volDataKey.nodeName] != string(csiMounter.plugin.host.GetNodeName()) {
 				t.Error("volume data file unexpected nodeName:", data[volDataKey.nodeName])
 			}
-			if data[volDataKey.driverMode] != string(csiMounter.driverMode) {
+			if data[volDataKey.csiVolumeMode] != string(csiMounter.csiVolumeMode) {
-				t.Error("volume data file unexpected driverMode:", data[volDataKey.driverMode])
+				t.Error("volume data file unexpected csiVolumeMode:", data[volDataKey.csiVolumeMode])
 			}
 		})
 	}

test/e2e/storage/external/external.go

@@ -40,6 +40,7 @@ import (
 
 // List of testSuites to be executed for each external driver.
 var csiTestSuites = []func() testsuites.TestSuite{
+	testsuites.InitEphemeralTestSuite,
 	testsuites.InitMultiVolumeTestSuite,
 	testsuites.InitProvisioningTestSuite,
 	testsuites.InitSnapshottableTestSuite,
@@ -128,6 +129,9 @@ var _ testsuites.DynamicPVTestDriver = &driverDefinition{}
 // Same for snapshotting.
 var _ testsuites.SnapshottableTestDriver = &driverDefinition{}
 
+// And for ephemeral volumes.
+var _ testsuites.EphemeralTestDriver = &driverDefinition{}
+
 // runtime.DecodeInto needs a runtime.Object but doesn't do any
 // deserialization of it and therefore none of the methods below need
 // an implementation.
@@ -174,6 +178,15 @@ type driverDefinition struct {
 		// TODO (?): load from file
 	}
 
+	// InlineVolumeAttributes defines one or more set of attributes for
+	// use as inline ephemeral volumes. At least one set of attributes
+	// has to be defined to enable testing of inline ephemeral volumes.
+	// If a test needs more volumes than defined, some of the defined
+	// volumes will be used multiple times.
+	//
+	// DriverInfo.Name is used as name of the driver in the inline volume.
+	InlineVolumeAttributes []map[string]string
+
 	// ClaimSize defines the desired size of dynamically
 	// provisioned volumes. Default is "5GiB".
 	ClaimSize string
@@ -206,6 +219,8 @@ func (d *driverDefinition) SkipUnsupportedTest(pattern testpatterns.TestPattern)
 		if d.StorageClass.FromName || d.StorageClass.FromFile != "" {
 			supported = true
 		}
+	case testpatterns.CSIInlineVolume:
+		supported = len(d.InlineVolumeAttributes) != 0
 	}
 	if !supported {
 		framework.Skipf("Driver %q does not support volume type %q - skipping", d.DriverInfo.Name, pattern.VolType)
@@ -278,6 +293,17 @@ func (d *driverDefinition) GetClaimSize() string {
 	return d.ClaimSize
 }
 
+func (d *driverDefinition) GetVolumeAttributes(config *testsuites.PerTestConfig, volumeNumber int) map[string]string {
+	if len(d.InlineVolumeAttributes) == 0 {
+		framework.Skipf("%s does not have any InlineVolumeAttributes defined", d.DriverInfo.Name)
+	}
+	return d.InlineVolumeAttributes[volumeNumber%len(d.InlineVolumeAttributes)]
+}
+
+func (d *driverDefinition) GetCSIDriverName(config *testsuites.PerTestConfig) string {
+	return d.DriverInfo.Name
+}
+
 func (d *driverDefinition) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
 	config := &testsuites.PerTestConfig{
 		Driver:         d,

test/e2e/storage/testpatterns/testpattern.go

@@ -44,6 +44,8 @@ var (
 	PreprovisionedPV TestVolType = "PreprovisionedPV"
 	// DynamicPV represents a volume type for dynamic provisioned Persistent Volume
 	DynamicPV TestVolType = "DynamicPV"
+	// CSIInlineVolume represents a volume type that is defined inline and provided by a CSI driver.
+	CSIInlineVolume TestVolType = "CSIInlineVolume"
 )
 
 // TestSnapshotType represents a snapshot type to be tested in a TestSuite

test/e2e/storage/testsuites/base.go

@@ -134,6 +134,8 @@ func skipUnsupportedTest(driver TestDriver, pattern testpatterns.TestPattern) {
 			_, isSupported = driver.(PreprovisionedPVTestDriver)
 		case testpatterns.DynamicPV:
 			_, isSupported = driver.(DynamicPVTestDriver)
+		case testpatterns.CSIInlineVolume:
+			_, isSupported = driver.(EphemeralTestDriver)
 		default:
 			isSupported = false
 		}

test/e2e/storage/testsuites/ephemeral.go

@@ -0,0 +1,230 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"fmt"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	"k8s.io/kubernetes/test/e2e/framework/volume"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+)
+
+type ephemeralTestSuite struct {
+	tsInfo TestSuiteInfo
+}
+
+var _ TestSuite = &ephemeralTestSuite{}
+
+// InitEphemeralTestSuite returns ephemeralTestSuite that implements TestSuite interface
+func InitEphemeralTestSuite() TestSuite {
+	return &ephemeralTestSuite{
+		tsInfo: TestSuiteInfo{
+			name: "ephemeral [Feature:CSIInlineVolume]",
+			testPatterns: []testpatterns.TestPattern{
+				{
+					Name:    "inline ephemeral CSI volume",
+					VolType: testpatterns.CSIInlineVolume,
+				},
+			},
+		},
+	}
+}
+
+func (p *ephemeralTestSuite) getTestSuiteInfo() TestSuiteInfo {
+	return p.tsInfo
+}
+
+func (p *ephemeralTestSuite) defineTests(driver TestDriver, pattern testpatterns.TestPattern) {
+	type local struct {
+		config      *PerTestConfig
+		testCleanup func()
+
+		testCase *EphemeralTest
+	}
+	var (
+		dInfo   = driver.GetDriverInfo()
+		eDriver EphemeralTestDriver
+		l       local
+	)
+
+	ginkgo.BeforeEach(func() {
+		ok := false
+		eDriver, ok = driver.(EphemeralTestDriver)
+		if !ok {
+			framework.Skipf("Driver %s doesn't support ephemeral inline volumes -- skipping", dInfo.Name)
+		}
+	})
+
+	// This intentionally comes after checking the preconditions because it
+	// registers its own BeforeEach which creates the namespace. Beware that it
+	// also registers an AfterEach which renders f unusable. Any code using
+	// f must run inside an It or Context callback.
+	f := framework.NewDefaultFramework("ephemeral")
+
+	init := func() {
+		l = local{}
+
+		// Now do the more expensive test initialization.
+		l.config, l.testCleanup = driver.PrepareTest(f)
+		l.testCase = &EphemeralTest{
+			Client:     l.config.Framework.ClientSet,
+			Namespace:  f.Namespace.Name,
+			DriverName: eDriver.GetCSIDriverName(l.config),
+			Node:       framework.NodeSelection{Name: l.config.ClientNodeName},
+			GetVolumeAttributes: func(volumeNumber int) map[string]string {
+				return eDriver.GetVolumeAttributes(l.config, volumeNumber)
+			},
+		}
+	}
+
+	cleanup := func() {
+		if l.testCleanup != nil {
+			l.testCleanup()
+			l.testCleanup = nil
+		}
+	}
+
+	ginkgo.It("should create inline ephemeral volume", func() {
+		init()
+		defer cleanup()
+
+		l.testCase.TestEphemeral()
+	})
+}
+
+// EphemeralTest represents parameters to be used by tests for inline volumes.
+// Not all parameters are used by all tests.
+type EphemeralTest struct {
+	Client     clientset.Interface
+	Namespace  string
+	DriverName string
+	Node       framework.NodeSelection
+
+	// GetVolumeAttributes returns the volume attributes for a
+	// certain inline ephemeral volume, enumerated starting with
+	// #0. Some tests might require more than one volume. They can
+	// all be the same or different, depending what the driver supports
+	// and/or wants to test.
+	GetVolumeAttributes func(volumeNumber int) map[string]string
+
+	// RunningPodCheck is invoked while a pod using an inline volume is running.
+	// It can execute additional checks on the pod and its volume(s). Any data
+	// returned by it is passed to StoppedPodCheck.
+	RunningPodCheck func(pod *v1.Pod) interface{}
+
+	// StoppedPodCheck is invoked after ensuring that the pod is gone.
+	// It is passed the data gather by RunningPodCheck or nil if that
+	// isn't defined and then can do additional checks on the node,
+	// like for example verifying that the ephemeral volume was really
+	// removed. How to do such a check is driver-specific and not
+	// covered by the generic storage test suite.
+	StoppedPodCheck func(nodeName string, runningPodData interface{})
+}
+
+// TestEphemeral tests pod creation with one ephemeral volume.
+func (t EphemeralTest) TestEphemeral() {
+	client := t.Client
+	gomega.Expect(client).NotTo(gomega.BeNil(), "EphemeralTest.Client is required")
+	gomega.Expect(t.GetVolumeAttributes).NotTo(gomega.BeNil(), "EphemeralTest.GetVolumeAttributes is required")
+	gomega.Expect(t.DriverName).NotTo(gomega.BeEmpty(), "EphemeralTest.DriverName is required")
+
+	ginkgo.By(fmt.Sprintf("checking the requested inline volume exists in the pod running on node %+v", t.Node))
+	command := "mount | grep /mnt/test"
+	pod := StartInPodWithInlineVolume(client, t.Namespace, "inline-volume-tester", command,
+		v1.CSIVolumeSource{
+			Driver:           t.DriverName,
+			VolumeAttributes: t.GetVolumeAttributes(0),
+		},
+		t.Node)
+	defer func() {
+		// pod might be nil now.
+		StopPod(client, pod)
+	}()
+	framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceSlow(client, pod.Name, pod.Namespace), "waiting for pod with inline volume")
+	runningPod, err := client.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "get pod")
+	actualNodeName := runningPod.Spec.NodeName
+
+	// Run the checker of the running pod.
+	var runningPodData interface{}
+	if t.RunningPodCheck != nil {
+		runningPodData = t.RunningPodCheck(pod)
+	}
+
+	StopPod(client, pod)
+	pod = nil // Don't stop twice.
+
+	if t.StoppedPodCheck != nil {
+		t.StoppedPodCheck(actualNodeName, runningPodData)
+	}
+}
+
+// StartInPodWithInlineVolume starts a command in a pod with given volume mounted to /mnt/test directory.
+// The caller is responsible for checking the pod and deleting it.
+func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command string, csiVolume v1.CSIVolumeSource, node framework.NodeSelection) *v1.Pod {
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: podName + "-",
+			Labels: map[string]string{
+				"app": podName,
+			},
+		},
+		Spec: v1.PodSpec{
+			NodeName:     node.Name,
+			NodeSelector: node.Selector,
+			Affinity:     node.Affinity,
+			Containers: []v1.Container{
+				{
+					Name:    "csi-volume-tester",
+					Image:   volume.GetTestImage(framework.BusyBoxImage),
+					Command: volume.GenerateScriptCmd(command),
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "my-volume",
+							MountPath: "/mnt/test",
+						},
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+			Volumes: []v1.Volume{
+				{
+					Name: "my-volume",
+					VolumeSource: v1.VolumeSource{
+						CSI: &csiVolume,
+					},
+				},
+			},
+		},
+	}
+
+	pod, err := c.CoreV1().Pods(ns).Create(pod)
+	framework.ExpectNoError(err, "failed to create pod")
+	return pod
+}

test/e2e/storage/testsuites/testdriver.go

@@ -101,6 +101,23 @@ type DynamicPVTestDriver interface {
 	GetClaimSize() string
 }
 
+// EphemeralTestDriver represents an interface for a TestDriver that supports ephemeral inline volumes.
+type EphemeralTestDriver interface {
+	TestDriver
+
+	// GetVolumeAttributes returns the volume attributes for a
+	// certain inline ephemeral volume, enumerated starting with
+	// #0. Some tests might require more than one volume. They can
+	// all be the same or different, depending what the driver supports
+	// and/or wants to test.
+	GetVolumeAttributes(config *PerTestConfig, volumeNumber int) map[string]string
+
+	// GetCSIDriverName returns the name that was used when registering with
+	// kubelet. Depending on how the driver was deployed, this can be different
+	// from DriverInfo.Name.
+	GetCSIDriverName(config *PerTestConfig) string
+}
+
 // SnapshottableTestDriver represents an interface for a TestDriver that supports DynamicSnapshot
 type SnapshottableTestDriver interface {
 	TestDriver