Merge pull request #69036 from wackxu/snapshottest

add e2e test for snapshot

test/e2e/framework/util.go

@@ -186,6 +186,9 @@ const (
 	// restart before test is considered failed.
 	RestartPodReadyAgainTimeout = 5 * time.Minute
 
+	// How long for snapshot to create snapshotContent
+	SnapshotCreateTimeout = 5 * time.Minute
+
 	// Number of objects that gc can delete in a second.
 	// GC issues 2 requestes for single delete.
 	gcThroughput = 10

test/e2e/storage/csi_volumes.go

@@ -62,6 +62,7 @@ var csiTestSuites = []func() testsuites.TestSuite{
 	testsuites.InitVolumeModeTestSuite,
 	testsuites.InitSubPathTestSuite,
 	testsuites.InitProvisioningTestSuite,
+	testsuites.InitSnapshottableTestSuite,
 }
 
 func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {

test/e2e/storage/drivers/BUILD

@@ -15,6 +15,7 @@ go_library(
         "//staging/src/k8s.io/api/storage/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",

test/e2e/storage/drivers/csi.go

@@ -41,6 +41,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
@@ -60,7 +61,7 @@ type hostpathCSIDriver struct {
 	manifests  []string
 }
 
-func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests ...string) testsuites.TestDriver {
+func initHostPathCSIDriver(name string, config testsuites.TestConfig, capabilities map[testsuites.Capability]bool, manifests ...string) testsuites.TestDriver {
 	return &hostpathCSIDriver{
 		driverInfo: testsuites.DriverInfo{
 			Name:        name,
@@ -69,10 +70,7 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
-			Capabilities: map[testsuites.Capability]bool{
+			Capabilities: capabilities,
-				testsuites.CapPersistence: true,
-			},
-
 			Config:       config,
 		},
 		manifests: manifests,
@@ -81,15 +79,19 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests
 
 var _ testsuites.TestDriver = &hostpathCSIDriver{}
 var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{}
+var _ testsuites.SnapshottableTestDriver = &hostpathCSIDriver{}
 
 // InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface
 func InitHostPathCSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
 	return initHostPathCSIDriver("csi-hostpath", config,
+		map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapDataSource: true},
 		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
 	)
@@ -111,6 +113,15 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *stor
 	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
 }
 
+func (h *hostpathCSIDriver) GetSnapshotClass() *unstructured.Unstructured {
+	snapshotter := testsuites.GetUniqueDriverName(h)
+	parameters := map[string]string{}
+	ns := h.driverInfo.Config.Framework.Namespace.Name
+	suffix := fmt.Sprintf("%s-vsc", snapshotter)
+
+	return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix)
+}
+
 func (h *hostpathCSIDriver) GetClaimSize() string {
 	return "5Gi"
 }
@@ -133,6 +144,7 @@ func (h *hostpathCSIDriver) CreateDriver() {
 		DriverContainerName:      "hostpath",
 		DriverContainerArguments: []string{"--drivername=csi-hostpath-" + f.UniqueName},
 		ProvisionerContainerName: "csi-provisioner",
+		SnapshotterContainerName: "csi-snapshotter",
 		NodeName:                 nodeName,
 	}
 	cleanup, err := h.driverInfo.Config.Framework.CreateFromManifests(func(item interface{}) error {
@@ -247,6 +259,7 @@ func (m *mockCSIDriver) CleanupDriver() {
 // InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests.
 func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
 	return initHostPathCSIDriver("csi-hostpath-v0", config,
+		map[testsuites.Capability]bool{testsuites.CapPersistence: true},
 		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",

test/e2e/storage/testpatterns/testpattern.go

@@ -17,7 +17,7 @@ limitations under the License.
 package testpatterns
 
 import (
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -45,6 +45,14 @@ var (
 	DynamicPV TestVolType = "DynamicPV"
 )
 
+// TestSnapshotType represents a snapshot type to be tested in a TestSuite
+type TestSnapshotType string
+
+var (
+	// DynamicCreatedSnapshot represents a snapshot type for dynamic created snapshot
+	DynamicCreatedSnapshot TestSnapshotType = "DynamicSnapshot"
+)
+
 // TestPattern represents a combination of parameters to be tested in a TestSuite
 type TestPattern struct {
 	Name         string                  // Name of TestPattern
@@ -52,6 +60,7 @@ type TestPattern struct {
 	VolType      TestVolType             // Volume type of the volume
 	FsType       string                  // Fstype of the volume
 	VolMode      v1.PersistentVolumeMode // PersistentVolumeMode of the volume
+	SnapshotType TestSnapshotType        // Snapshot type of the snapshot
 }
 
 var (
@@ -165,4 +174,12 @@ var (
 		VolType: DynamicPV,
 		VolMode: v1.PersistentVolumeBlock,
 	}
+
+	// Definitions for snapshot case
+
+	// DynamicSnapshot is TestPattern for "Dynamic snapshot"
+	DynamicSnapshot = TestPattern{
+		Name:         "Dynamic Snapshot",
+		SnapshotType: DynamicCreatedSnapshot,
+	}
 )

test/e2e/storage/testsuites/BUILD

@@ -6,6 +6,7 @@ go_library(
         "base.go",
         "driveroperations.go",
         "provisioning.go",
+        "snapshottable.go",
         "subpath.go",
         "testdriver.go",
         "volume_io.go",
@@ -20,11 +21,14 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/storage/testpatterns:go_default_library",

test/e2e/storage/testsuites/base.go

@@ -22,11 +22,13 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -79,15 +81,31 @@ func RunTestSuite(f *framework.Framework, driver TestDriver, tsInits []func() Te
 // skipUnsupportedTest will skip tests if the combination of driver, testsuite, and testpattern
 // is not suitable to be tested.
 // Whether it needs to be skipped is checked by following steps:
-// 1. Check if Whether volType is supported by driver from its interface
+// 1. Check if Whether SnapshotType is supported by driver from its interface
-// 2. Check if fsType is supported
+// 2. Check if Whether volType is supported by driver from its interface
-// 3. Check with driver specific logic
+// 3. Check if fsType is supported
-// 4. Check with testSuite specific logic
+// 4. Check with driver specific logic
+// 5. Check with testSuite specific logic
 func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpatterns.TestPattern) {
 	dInfo := driver.GetDriverInfo()
-
-	// 1. Check if Whether volType is supported by driver from its interface
 	var isSupported bool
+
+	// 1. Check if Whether SnapshotType is supported by driver from its interface
+	// if isSupported, so it must be a snapshot test case, we just return.
+	if len(pattern.SnapshotType) > 0 {
+		switch pattern.SnapshotType {
+		case testpatterns.DynamicCreatedSnapshot:
+			_, isSupported = driver.(SnapshottableTestDriver)
+		default:
+			isSupported = false
+		}
+		if !isSupported {
+			framework.Skipf("Driver %s doesn't support snapshot type %v -- skipping", dInfo.Name, pattern.SnapshotType)
+		}
+		return
+	}
+
+	// 2. Check if Whether volType is supported by driver from its interface
 	switch pattern.VolType {
 	case testpatterns.InlineVolume:
 		_, isSupported = driver.(InlineVolumeTestDriver)
@@ -103,7 +121,7 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern
 		framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType)
 	}
 
-	// 2. Check if fsType is supported
+	// 3. Check if fsType is supported
 	if !dInfo.SupportedFsType.Has(pattern.FsType) {
 		framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.FsType)
 	}
@@ -111,10 +129,10 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern
 		framework.Skipf("Distro doesn't support xfs -- skipping")
 	}
 
-	// 3. Check with driver specific logic
+	// 4. Check with driver specific logic
 	driver.SkipUnsupportedTest(pattern)
 
-	// 4. Check with testSuite specific logic
+	// 5. Check with testSuite specific logic
 	suite.skipUnsupportedTest(pattern, driver)
 }
 
@@ -349,3 +367,25 @@ func convertTestConfig(in *TestConfig) framework.VolumeTestConfig {
 		NodeSelector:   in.ClientNodeSelector,
 	}
 }
+
+func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.Unstructured {
+	snapshot := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       "VolumeSnapshot",
+			"apiVersion": snapshotAPIVersion,
+			"metadata": map[string]interface{}{
+				"generateName": "snapshot-",
+				"namespace":    ns,
+			},
+			"spec": map[string]interface{}{
+				"snapshotClassName": snapshotClassName,
+				"source": map[string]interface{}{
+					"name": claimName,
+					"kind": "PersistentVolumeClaim",
+				},
+			},
+		},
+	}
+
+	return snapshot
+}

test/e2e/storage/testsuites/driveroperations.go

@@ -21,6 +21,7 @@ import (
 
 	storagev1 "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
 )
@@ -95,6 +96,30 @@ func GetStorageClass(
 	}
 }
 
+// GetSnapshotClass constructs a new SnapshotClass instance
+// with a unique name that is based on namespace + suffix.
+func GetSnapshotClass(
+	snapshotter string,
+	parameters map[string]string,
+	ns string,
+	suffix string,
+) *unstructured.Unstructured {
+	snapshotClass := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       "VolumeSnapshotClass",
+			"apiVersion": snapshotAPIVersion,
+			"metadata": map[string]interface{}{
+				// Name must be unique, so let's base it on namespace name
+				"name": ns + "-" + suffix,
+			},
+			"snapshotter": snapshotter,
+			"parameters":  parameters,
+		},
+	}
+
+	return snapshotClass
+}
+
 // GetUniqueDriverName returns unique driver name that can be used parallelly in tests
 func GetUniqueDriverName(driver TestDriver) string {
 	return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Config.Framework.UniqueName)

test/e2e/storage/testsuites/provisioning.go

@@ -23,13 +23,14 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	apierrs "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/resource"
-
 	"k8s.io/api/core/v1"
 	storage "k8s.io/api/storage/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
@@ -90,8 +91,10 @@ func createProvisioningTestInput(driver TestDriver, pattern testpatterns.TestPat
 			ExpectedSize: resource.claimSize,
 		},
 		cs:    driver.GetDriverInfo().Config.Framework.ClientSet,
+		dc:    driver.GetDriverInfo().Config.Framework.DynamicClient,
 		pvc:   resource.pvc,
 		sc:    resource.sc,
+		vsc:   resource.vsc,
 		dInfo: driver.GetDriverInfo(),
 	}
 
@@ -139,6 +142,8 @@ type provisioningTestResource struct {
 	claimSize string
 	sc        *storage.StorageClass
 	pvc       *v1.PersistentVolumeClaim
+	// follow parameter is used to test provision volume from snapshot
+	vsc *unstructured.Unstructured
 }
 
 var _ TestResource = &provisioningTestResource{}
@@ -157,6 +162,9 @@ func (p *provisioningTestResource) setupResource(driver TestDriver, pattern test
 			p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
 			p.pvc.Spec.StorageClassName = &p.sc.Name
 			framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc)
+			if sDriver, ok := driver.(SnapshottableTestDriver); ok {
+				p.vsc = sDriver.GetSnapshotClass()
+			}
 		}
 	default:
 		framework.Failf("Dynamic Provision test doesn't support: %s", pattern.VolType)
@@ -169,8 +177,10 @@ func (p *provisioningTestResource) cleanupResource(driver TestDriver, pattern te
 type provisioningTestInput struct {
 	testCase StorageClassTest
 	cs       clientset.Interface
+	dc       dynamic.Interface
 	pvc      *v1.PersistentVolumeClaim
 	sc       *storage.StorageClass
+	vsc      *unstructured.Unstructured
 	dInfo    *DriverInfo
 }
 
@@ -198,6 +208,19 @@ func testProvisioning(input *provisioningTestInput) {
 		input.pvc.Spec.VolumeMode = &block
 		TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
 	})
+
+	It("should provision storage with snapshot data source [Feature:VolumeSnapshotDataSource]", func() {
+		if !input.dInfo.Capabilities[CapDataSource] {
+			framework.Skipf("Driver %q does not support populate data from snapshot - skipping", input.dInfo.Name)
+		}
+
+		input.testCase.SkipWriteReadCheck = true
+		dataSource, cleanupFunc := prepareDataSourceForProvisioning(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc)
+		defer cleanupFunc()
+
+		input.pvc.Spec.DataSource = dataSource
+		TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc)
+	})
 }
 
 // TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest and storageClass
@@ -205,7 +228,11 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
 	var err error
 	if class != nil {
 		By("creating a StorageClass " + class.Name)
-		class, err = client.StorageV1().StorageClasses().Create(class)
+		_, err = client.StorageV1().StorageClasses().Create(class)
+		// The "should provision storage with snapshot data source" test already has created the class.
+		// TODO: make class creation optional and remove the IsAlreadyExists exception
+		Expect(err == nil || apierrs.IsAlreadyExists(err)).To(Equal(true))
+		class, err = client.StorageV1().StorageClasses().Get(class.Name, metav1.GetOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		defer func() {
 			framework.Logf("deleting storage class %s", class.Name)
@@ -268,6 +295,12 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
 		Expect(err).NotTo(HaveOccurred())
 	}
 
+	if claim.Spec.DataSource != nil {
+		By("checking the created volume whether has the pre-populated data")
+		command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace)
+		runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, command, t.NodeSelector, t.ExpectUnschedulable)
+	}
+
 	if !t.SkipWriteReadCheck {
 		// We start two pods:
 		// - The first writes 'hello word' to the /mnt/test (= the volume).
@@ -288,6 +321,7 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla
 		By("checking the created volume is readable and retains data")
 		runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, "grep 'hello world' /mnt/test/data", t.NodeSelector, t.ExpectUnschedulable)
 	}
+
 	By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name))
 	framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil))
 
@@ -466,3 +500,76 @@ func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeCl
 		Expect(claim.Status.Phase).To(Equal(v1.ClaimPending))
 	}
 }
+
+func prepareDataSourceForProvisioning(
+	t StorageClassTest,
+	client clientset.Interface,
+	dynamicClient dynamic.Interface,
+	initClaim *v1.PersistentVolumeClaim,
+	class *storage.StorageClass,
+	snapshotClass *unstructured.Unstructured,
+) (*v1.TypedLocalObjectReference, func()) {
+	var err error
+	if class != nil {
+		By("[Initialize dataSource]creating a StorageClass " + class.Name)
+		_, err = client.StorageV1().StorageClasses().Create(class)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	By("[Initialize dataSource]creating a initClaim")
+	updatedClaim, err := client.CoreV1().PersistentVolumeClaims(initClaim.Namespace).Create(initClaim)
+	Expect(err).NotTo(HaveOccurred())
+	err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, updatedClaim.Namespace, updatedClaim.Name, framework.Poll, framework.ClaimProvisionTimeout)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("[Initialize dataSource]checking the initClaim")
+	// Get new copy of the initClaim
+	_, err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Get(updatedClaim.Name, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	// write namespace to the /mnt/test (= the volume).
+	By("[Initialize dataSource]write data to volume")
+	command := fmt.Sprintf("echo '%s' > /mnt/test/initialData", updatedClaim.GetNamespace())
+	runInPodWithVolume(client, updatedClaim.Namespace, updatedClaim.Name, t.NodeName, command, t.NodeSelector, t.ExpectUnschedulable)
+
+	By("[Initialize dataSource]creating a SnapshotClass")
+	snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{})
+
+	By("[Initialize dataSource]creating a snapshot")
+	snapshot := getSnapshot(updatedClaim.Name, updatedClaim.Namespace, snapshotClass.GetName())
+	snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(updatedClaim.Namespace).Create(snapshot, metav1.CreateOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("[Initialize dataSource]checking the snapshot")
+	// Get new copy of the snapshot
+	snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	group := "snapshot.storage.k8s.io"
+	dataSourceRef := &v1.TypedLocalObjectReference{
+		APIGroup: &group,
+		Kind:     "VolumeSnapshot",
+		Name:     snapshot.GetName(),
+	}
+
+	cleanupFunc := func() {
+		framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName())
+		err = dynamicClient.Resource(snapshotGVR).Namespace(updatedClaim.Namespace).Delete(snapshot.GetName(), nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			framework.Failf("Error deleting snapshot %q. Error: %v", snapshot.GetName(), err)
+		}
+
+		framework.Logf("deleting initClaim %q/%q", updatedClaim.Namespace, updatedClaim.Name)
+		err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Delete(updatedClaim.Name, nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			framework.Failf("Error deleting initClaim %q. Error: %v", updatedClaim.Name, err)
+		}
+
+		framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName())
+		framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil))
+	}
+
+	return dataSourceRef, cleanupFunc
+}

test/e2e/storage/testsuites/snapshottable.go

@@ -0,0 +1,323 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/api/core/v1"
+	storage "k8s.io/api/storage/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+)
+
+// snapshot CRD api group
+const snapshotGroup = "snapshot.storage.k8s.io"
+
+// snapshot CRD api version
+const snapshotAPIVersion = "snapshot.storage.k8s.io/v1alpha1"
+
+var (
+	snapshotGVR        = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshots"}
+	snapshotClassGVR   = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotclasses"}
+	snapshotContentGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotcontents"}
+)
+
+type SnapshotClassTest struct {
+	Name                 string
+	CloudProviders       []string
+	Snapshotter          string
+	Parameters           map[string]string
+	NodeName             string
+	NodeSelector         map[string]string // NodeSelector for the pod
+	SnapshotContentCheck func(snapshotContent *unstructured.Unstructured) error
+}
+
+type snapshottableTestSuite struct {
+	tsInfo TestSuiteInfo
+}
+
+var _ TestSuite = &snapshottableTestSuite{}
+
+// InitSnapshottableTestSuite returns snapshottableTestSuite that implements TestSuite interface
+func InitSnapshottableTestSuite() TestSuite {
+	return &snapshottableTestSuite{
+		tsInfo: TestSuiteInfo{
+			name: "snapshottable",
+			testPatterns: []testpatterns.TestPattern{
+				testpatterns.DynamicSnapshot,
+			},
+		},
+	}
+}
+
+func (s *snapshottableTestSuite) getTestSuiteInfo() TestSuiteInfo {
+	return s.tsInfo
+}
+
+func (s *snapshottableTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) {
+}
+
+func createSnapshottableTestInput(driver TestDriver, pattern testpatterns.TestPattern) (snapshottableTestResource, snapshottableTestInput) {
+	// Setup test resource for driver and testpattern
+	resource := snapshottableTestResource{}
+	resource.setupResource(driver, pattern)
+
+	input := snapshottableTestInput{
+		testCase: SnapshotClassTest{},
+		cs:       driver.GetDriverInfo().Config.Framework.ClientSet,
+		dc:       driver.GetDriverInfo().Config.Framework.DynamicClient,
+		pvc:      resource.pvc,
+		sc:       resource.sc,
+		vsc:      resource.vsc,
+		dInfo:    driver.GetDriverInfo(),
+	}
+
+	if driver.GetDriverInfo().Config.ClientNodeName != "" {
+		input.testCase.NodeName = driver.GetDriverInfo().Config.ClientNodeName
+	}
+
+	return resource, input
+}
+
+func (s *snapshottableTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) {
+	Context(getTestNameStr(s, pattern), func() {
+		var (
+			resource     snapshottableTestResource
+			input        snapshottableTestInput
+			needsCleanup bool
+		)
+
+		BeforeEach(func() {
+			needsCleanup = false
+			// Skip unsupported tests to avoid unnecessary resource initialization
+			skipUnsupportedTest(s, driver, pattern)
+			needsCleanup = true
+
+			// Create test input
+			resource, input = createSnapshottableTestInput(driver, pattern)
+		})
+
+		AfterEach(func() {
+			if needsCleanup {
+				resource.cleanupResource(driver, pattern)
+			}
+		})
+
+		// Ginkgo's "Global Shared Behaviors" require arguments for a shared function
+		// to be a single struct and to be passed as a pointer.
+		// Please see https://onsi.github.io/ginkgo/#global-shared-behaviors for details.
+		testSnapshot(&input)
+	})
+}
+
+type snapshottableTestResource struct {
+	driver    TestDriver
+	claimSize string
+
+	sc  *storage.StorageClass
+	pvc *v1.PersistentVolumeClaim
+	// volume snapshot class
+	vsc *unstructured.Unstructured
+}
+
+var _ TestResource = &snapshottableTestResource{}
+
+func (s *snapshottableTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) {
+	// Setup snapshottableTest resource
+	switch pattern.SnapshotType {
+	case testpatterns.DynamicCreatedSnapshot:
+		if dDriver, ok := driver.(DynamicPVTestDriver); ok {
+			s.sc = dDriver.GetDynamicProvisionStorageClass("")
+			if s.sc == nil {
+				framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
+			}
+			s.driver = driver
+			s.claimSize = dDriver.GetClaimSize()
+			s.pvc = getClaim(s.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name)
+			s.pvc.Spec.StorageClassName = &s.sc.Name
+			framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", s.sc, s.pvc)
+
+			if sDriver, ok := driver.(SnapshottableTestDriver); ok {
+				s.vsc = sDriver.GetSnapshotClass()
+			}
+		}
+
+	default:
+		framework.Failf("Dynamic Snapshot test doesn't support: %s", pattern.SnapshotType)
+	}
+}
+
+func (s *snapshottableTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) {
+}
+
+type snapshottableTestInput struct {
+	testCase SnapshotClassTest
+	cs       clientset.Interface
+	dc       dynamic.Interface
+	pvc      *v1.PersistentVolumeClaim
+	sc       *storage.StorageClass
+	// volume snapshot class
+	vsc   *unstructured.Unstructured
+	dInfo *DriverInfo
+}
+
+func testSnapshot(input *snapshottableTestInput) {
+	It("should create snapshot with defaults", func() {
+		if input.dInfo.Name == "csi-hostpath-v0" {
+			framework.Skipf("skip test when using driver csi-hostpath-v0 - skipping")
+		}
+		TestCreateSnapshot(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc)
+	})
+}
+
+// TestCreateSnapshot tests dynamic creating snapshot with specified SnapshotClassTest and snapshotClass
+func TestCreateSnapshot(
+	t SnapshotClassTest,
+	client clientset.Interface,
+	dynamicClient dynamic.Interface,
+	claim *v1.PersistentVolumeClaim,
+	class *storage.StorageClass,
+	snapshotClass *unstructured.Unstructured,
+) *unstructured.Unstructured {
+	var err error
+	if class != nil {
+		By("creating a StorageClass " + class.Name)
+		class, err = client.StorageV1().StorageClasses().Create(class)
+		Expect(err).NotTo(HaveOccurred())
+		defer func() {
+			framework.Logf("deleting storage class %s", class.Name)
+			framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil))
+		}()
+	}
+
+	By("creating a claim")
+	claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
+	Expect(err).NotTo(HaveOccurred())
+	defer func() {
+		framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name)
+		// typically this claim has already been deleted
+		err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err)
+		}
+	}()
+	err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("checking the claim")
+	// Get new copy of the claim
+	claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	// Get the bound PV
+	pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	By("creating a SnapshotClass")
+	snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	defer func() {
+		framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName())
+		framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil))
+	}()
+
+	By("creating a snapshot")
+	snapshot := getSnapshot(claim.Name, claim.Namespace, snapshotClass.GetName())
+
+	snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Create(snapshot, metav1.CreateOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	defer func() {
+		framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName())
+		// typically this snapshot has already been deleted
+		err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Delete(snapshot.GetName(), nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			framework.Failf("Error deleting snapshot %q. Error: %v", claim.Name, err)
+		}
+	}()
+	err = WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("checking the snapshot")
+	// Get new copy of the snapshot
+	snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	// Get the bound snapshotContent
+	snapshotSpec := snapshot.Object["spec"].(map[string]interface{})
+	snapshotContentName := snapshotSpec["snapshotContentName"].(string)
+	snapshotContent, err := dynamicClient.Resource(snapshotContentGVR).Get(snapshotContentName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	snapshotContentSpec := snapshotContent.Object["spec"].(map[string]interface{})
+	volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{})
+	persistentVolumeRef := snapshotContentSpec["persistentVolumeRef"].(map[string]interface{})
+
+	// Check SnapshotContent properties
+	By("checking the SnapshotContent")
+	Expect(snapshotContentSpec["snapshotClassName"]).To(Equal(snapshotClass.GetName()))
+	Expect(volumeSnapshotRef["name"]).To(Equal(snapshot.GetName()))
+	Expect(volumeSnapshotRef["namespace"]).To(Equal(snapshot.GetNamespace()))
+	Expect(persistentVolumeRef["name"]).To(Equal(pv.Name))
+
+	// Run the checker
+	if t.SnapshotContentCheck != nil {
+		err = t.SnapshotContentCheck(snapshotContent)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	return snapshotContent
+}
+
+// WaitForSnapshotReady waits for a VolumeSnapshot to be ready to use or until timeout occurs, whichever comes first.
+func WaitForSnapshotReady(c dynamic.Interface, ns string, snapshotName string, Poll, timeout time.Duration) error {
+	framework.Logf("Waiting up to %v for VolumeSnapshot %s to become ready", timeout, snapshotName)
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
+		snapshot, err := c.Resource(snapshotGVR).Namespace(ns).Get(snapshotName, metav1.GetOptions{})
+		if err != nil {
+			framework.Logf("Failed to get claim %q, retrying in %v. Error: %v", snapshotName, Poll, err)
+			continue
+		} else {
+			status := snapshot.Object["status"]
+			if status == nil {
+				framework.Logf("VolumeSnapshot %s found but is not ready.", snapshotName)
+				continue
+			}
+			value := status.(map[string]interface{})
+			if value["readyToUse"] == true {
+				framework.Logf("VolumeSnapshot %s found and is ready", snapshotName, time.Since(start))
+				return nil
+			} else if value["ready"] == true {
+				framework.Logf("VolumeSnapshot %s found and is ready", snapshotName, time.Since(start))
+				return nil
+			} else {
+				framework.Logf("VolumeSnapshot %s found but is not ready.", snapshotName)
+			}
+		}
+	}
+	return fmt.Errorf("VolumeSnapshot %s is not ready within %v", snapshotName, timeout)
+}

test/e2e/storage/testsuites/testdriver.go

@@ -19,6 +19,7 @@ package testsuites
 import (
 	"k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
@@ -79,6 +80,14 @@ type DynamicPVTestDriver interface {
 	GetClaimSize() string
 }
 
+// SnapshottableTestDriver represents an interface for a TestDriver that supports DynamicSnapshot
+type SnapshottableTestDriver interface {
+	TestDriver
+	// GetSnapshotClass returns a SnapshotClass to create snapshot.
+	// It will return nil, if the TestDriver doesn't support it.
+	GetSnapshotClass() *unstructured.Unstructured
+}
+
 // Capability represents a feature that a volume plugin supports
 type Capability string
 
@@ -87,6 +96,7 @@ const (
 	CapBlock       Capability = "block"       // raw block mode
 	CapFsGroup     Capability = "fsGroup"     // volume ownership via fsGroup
 	CapExec        Capability = "exec"        // exec a file in the volume
+	CapDataSource  Capability = "dataSource"  // support populate data from snapshot
 )
 
 // DriverInfo represents a combination of parameters to be used in implementation of TestDriver

test/e2e/storage/utils/deployment.go

@@ -93,6 +93,10 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf
 				// Driver name is expected to be the same
 				// as the provisioner here.
 				container.Args = append(container.Args, "--provisioner="+o.NewDriverName)
+			case o.SnapshotterContainerName:
+				// Driver name is expected to be the same
+				// as the snapshotter here.
+				container.Args = append(container.Args, "--snapshotter="+o.NewDriverName)
 			}
 		}
 	}
@@ -145,6 +149,10 @@ type PatchCSIOptions struct {
 	// If non-empty, --provisioner with new name will be appended
 	// to the argument list.
 	ProvisionerContainerName string
+	// The name of the container which has the snapshotter binary.
+	// If non-empty, --snapshotter with new name will be appended
+	// to the argument list.
+	SnapshotterContainerName string
 	// If non-empty, all pods are forced to run on this node.
 	NodeName string
 }

test/e2e/testing-manifests/storage-csi/controller-role.yaml

@@ -1,3 +1,4 @@
-# Replaced by individual roles for external-attacher and external-provisioner:
+# Replaced by individual roles for external-attacher, external-provisioner and external-snapshotter:
 # - https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
 # - https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml
+# - https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md

@@ -0,0 +1 @@
+The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml

@@ -0,0 +1,65 @@
+# Together with the RBAC file for external-provisioner, this YAML file
+# contains all RBAC objects that are necessary to run external CSI
+# snapshotter.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - optionally rename the non-namespaced ClusterRole if there
+#   are conflicts with other deployments
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-snapshotter
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  # rename if there are conflicts
+  name: external-snapshotter-runner
+rules:
+- apiGroups: [""]
+  resources: ["persistentvolumes"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["persistentvolumeclaims"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["list", "watch", "create", "update", "patch"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+- apiGroups: ["snapshot.storage.k8s.io"]
+  resources: ["volumesnapshotclasses"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["snapshot.storage.k8s.io"]
+  resources: ["volumesnapshotcontents"]
+  verbs: ["create", "get", "list", "watch", "update", "delete"]
+- apiGroups: ["snapshot.storage.k8s.io"]
+  resources: ["volumesnapshots"]
+  verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["create", "list", "watch", "delete"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-snapshotter-role
+subjects:
+- kind: ServiceAccount
+  name: csi-snapshotter
+  # replace with non-default namespace name
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  # change the name also here if the ClusterRole gets renamed
+  name: external-snapshotter-runner
+  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/hostpath/README.md

@@ -1,5 +1,5 @@
 A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example,
 with some modifications:
 - serviceAccountName is used instead of the deprecated serviceAccount
-- the RBAC roles from driver-registrar, external-attacher and external-provisioner
+- the RBAC roles from driver-registrar, external-attacher, external-provisioner
-  are used
+  and external-snapshotter are used

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml

@@ -0,0 +1,48 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-snapshotter
+  labels:
+    app: csi-snapshotter
+spec:
+  selector:
+    app: csi-snapshotter
+  ports:
+  - name: dummy
+    port: 12345
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-snapshotter
+spec:
+  serviceName: "csi-snapshotter"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-snapshotter
+  template:
+    metadata:
+      labels:
+        app: csi-snapshotter
+    spec:
+      serviceAccount: csi-snapshotter
+      containers:
+      - name: csi-snapshotter
+        image: quay.io/k8scsi/csi-snapshotter:v1.0.1
+        args:
+        - "--csi-address=$(ADDRESS)"
+        - "--connection-timeout=15s"
+        env:
+        - name: ADDRESS
+          value: /csi/csi.sock
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: socket-dir
+          mountPath: /csi
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet/plugins/csi-hostpath
+          type: DirectoryOrCreate
+        name: socket-dir

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml

@@ -13,6 +13,9 @@ subjects:
   - kind: ServiceAccount
     name: csi-provisioner
     namespace: default
+  - kind: ServiceAccount
+    name: csi-snapshotter
+    namespace: default
 roleRef:
   kind: ClusterRole
   name: e2e-test-privileged-psp