diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 6759563750f..7256ab61eb3 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -186,6 +186,9 @@ const ( // restart before test is considered failed. RestartPodReadyAgainTimeout = 5 * time.Minute + // How long for snapshot to create snapshotContent + SnapshotCreateTimeout = 5 * time.Minute + // Number of objects that gc can delete in a second. // GC issues 2 requestes for single delete. gcThroughput = 10 diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index b091732f4d4..4886ae16afc 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -62,6 +62,7 @@ var csiTestSuites = []func() testsuites.TestSuite{ testsuites.InitVolumeModeTestSuite, testsuites.InitSubPathTestSuite, testsuites.InitProvisioningTestSuite, + testsuites.InitSnapshottableTestSuite, } func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern { diff --git a/test/e2e/storage/drivers/BUILD b/test/e2e/storage/drivers/BUILD index 4ab3eb8a5e1..6e763ddee6d 100644 --- a/test/e2e/storage/drivers/BUILD +++ b/test/e2e/storage/drivers/BUILD @@ -15,6 +15,7 @@ go_library( "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 6551854e643..c15e94c5f78 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -41,6 +41,7 @@ import ( . "github.com/onsi/ginkgo" storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testpatterns" @@ -60,7 +61,7 @@ type hostpathCSIDriver struct { manifests []string } -func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests ...string) testsuites.TestDriver { +func initHostPathCSIDriver(name string, config testsuites.TestConfig, capabilities map[testsuites.Capability]bool, manifests ...string) testsuites.TestDriver { return &hostpathCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: name, @@ -69,11 +70,8 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests SupportedFsType: sets.NewString( "", // Default fsType ), - Capabilities: map[testsuites.Capability]bool{ - testsuites.CapPersistence: true, - }, - - Config: config, + Capabilities: capabilities, + Config: config, }, manifests: manifests, } @@ -81,15 +79,19 @@ func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests var _ testsuites.TestDriver = &hostpathCSIDriver{} var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{} +var _ testsuites.SnapshottableTestDriver = &hostpathCSIDriver{} // InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface func InitHostPathCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { return initHostPathCSIDriver("csi-hostpath", config, + map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapDataSource: true}, "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", ) @@ -111,6 +113,15 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *stor return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } +func (h *hostpathCSIDriver) GetSnapshotClass() *unstructured.Unstructured { + snapshotter := testsuites.GetUniqueDriverName(h) + parameters := map[string]string{} + ns := h.driverInfo.Config.Framework.Namespace.Name + suffix := fmt.Sprintf("%s-vsc", snapshotter) + + return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix) +} + func (h *hostpathCSIDriver) GetClaimSize() string { return "5Gi" } @@ -133,6 +144,7 @@ func (h *hostpathCSIDriver) CreateDriver() { DriverContainerName: "hostpath", DriverContainerArguments: []string{"--drivername=csi-hostpath-" + f.UniqueName}, ProvisionerContainerName: "csi-provisioner", + SnapshotterContainerName: "csi-snapshotter", NodeName: nodeName, } cleanup, err := h.driverInfo.Config.Framework.CreateFromManifests(func(item interface{}) error { @@ -247,6 +259,7 @@ func (m *mockCSIDriver) CleanupDriver() { // InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests. func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver { return initHostPathCSIDriver("csi-hostpath-v0", config, + map[testsuites.Capability]bool{testsuites.CapPersistence: true}, "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", diff --git a/test/e2e/storage/testpatterns/testpattern.go b/test/e2e/storage/testpatterns/testpattern.go index 26b311697f9..a9fc2df37c2 100644 --- a/test/e2e/storage/testpatterns/testpattern.go +++ b/test/e2e/storage/testpatterns/testpattern.go @@ -17,7 +17,7 @@ limitations under the License. package testpatterns import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/kubernetes/test/e2e/framework" ) @@ -45,13 +45,22 @@ var ( DynamicPV TestVolType = "DynamicPV" ) +// TestSnapshotType represents a snapshot type to be tested in a TestSuite +type TestSnapshotType string + +var ( + // DynamicCreatedSnapshot represents a snapshot type for dynamic created snapshot + DynamicCreatedSnapshot TestSnapshotType = "DynamicSnapshot" +) + // TestPattern represents a combination of parameters to be tested in a TestSuite type TestPattern struct { - Name string // Name of TestPattern - FeatureTag string // featureTag for the TestSuite - VolType TestVolType // Volume type of the volume - FsType string // Fstype of the volume - VolMode v1.PersistentVolumeMode // PersistentVolumeMode of the volume + Name string // Name of TestPattern + FeatureTag string // featureTag for the TestSuite + VolType TestVolType // Volume type of the volume + FsType string // Fstype of the volume + VolMode v1.PersistentVolumeMode // PersistentVolumeMode of the volume + SnapshotType TestSnapshotType // Snapshot type of the snapshot } var ( @@ -165,4 +174,12 @@ var ( VolType: DynamicPV, VolMode: v1.PersistentVolumeBlock, } + + // Definitions for snapshot case + + // DynamicSnapshot is TestPattern for "Dynamic snapshot" + DynamicSnapshot = TestPattern{ + Name: "Dynamic Snapshot", + SnapshotType: DynamicCreatedSnapshot, + } ) diff --git a/test/e2e/storage/testsuites/BUILD b/test/e2e/storage/testsuites/BUILD index e0b5ad79864..9a692f0a435 100644 --- a/test/e2e/storage/testsuites/BUILD +++ b/test/e2e/storage/testsuites/BUILD @@ -6,6 +6,7 @@ go_library( "base.go", "driveroperations.go", "provisioning.go", + "snapshottable.go", "subpath.go", "testdriver.go", "volume_io.go", @@ -20,11 +21,14 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/storage/testpatterns:go_default_library", diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index d1325aee7c5..3c7e58d4ff1 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -22,11 +22,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilerrors "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -79,15 +81,31 @@ func RunTestSuite(f *framework.Framework, driver TestDriver, tsInits []func() Te // skipUnsupportedTest will skip tests if the combination of driver, testsuite, and testpattern // is not suitable to be tested. // Whether it needs to be skipped is checked by following steps: -// 1. Check if Whether volType is supported by driver from its interface -// 2. Check if fsType is supported -// 3. Check with driver specific logic -// 4. Check with testSuite specific logic +// 1. Check if Whether SnapshotType is supported by driver from its interface +// 2. Check if Whether volType is supported by driver from its interface +// 3. Check if fsType is supported +// 4. Check with driver specific logic +// 5. Check with testSuite specific logic func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() - - // 1. Check if Whether volType is supported by driver from its interface var isSupported bool + + // 1. Check if Whether SnapshotType is supported by driver from its interface + // if isSupported, so it must be a snapshot test case, we just return. + if len(pattern.SnapshotType) > 0 { + switch pattern.SnapshotType { + case testpatterns.DynamicCreatedSnapshot: + _, isSupported = driver.(SnapshottableTestDriver) + default: + isSupported = false + } + if !isSupported { + framework.Skipf("Driver %s doesn't support snapshot type %v -- skipping", dInfo.Name, pattern.SnapshotType) + } + return + } + + // 2. Check if Whether volType is supported by driver from its interface switch pattern.VolType { case testpatterns.InlineVolume: _, isSupported = driver.(InlineVolumeTestDriver) @@ -103,7 +121,7 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType) } - // 2. Check if fsType is supported + // 3. Check if fsType is supported if !dInfo.SupportedFsType.Has(pattern.FsType) { framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.FsType) } @@ -111,10 +129,10 @@ func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpattern framework.Skipf("Distro doesn't support xfs -- skipping") } - // 3. Check with driver specific logic + // 4. Check with driver specific logic driver.SkipUnsupportedTest(pattern) - // 4. Check with testSuite specific logic + // 5. Check with testSuite specific logic suite.skipUnsupportedTest(pattern, driver) } @@ -349,3 +367,25 @@ func convertTestConfig(in *TestConfig) framework.VolumeTestConfig { NodeSelector: in.ClientNodeSelector, } } + +func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.Unstructured { + snapshot := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "VolumeSnapshot", + "apiVersion": snapshotAPIVersion, + "metadata": map[string]interface{}{ + "generateName": "snapshot-", + "namespace": ns, + }, + "spec": map[string]interface{}{ + "snapshotClassName": snapshotClassName, + "source": map[string]interface{}{ + "name": claimName, + "kind": "PersistentVolumeClaim", + }, + }, + }, + } + + return snapshot +} diff --git a/test/e2e/storage/testsuites/driveroperations.go b/test/e2e/storage/testsuites/driveroperations.go index 1e233b48387..e0adbc06007 100644 --- a/test/e2e/storage/testsuites/driveroperations.go +++ b/test/e2e/storage/testsuites/driveroperations.go @@ -21,6 +21,7 @@ import ( storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testpatterns" ) @@ -95,6 +96,30 @@ func GetStorageClass( } } +// GetSnapshotClass constructs a new SnapshotClass instance +// with a unique name that is based on namespace + suffix. +func GetSnapshotClass( + snapshotter string, + parameters map[string]string, + ns string, + suffix string, +) *unstructured.Unstructured { + snapshotClass := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "VolumeSnapshotClass", + "apiVersion": snapshotAPIVersion, + "metadata": map[string]interface{}{ + // Name must be unique, so let's base it on namespace name + "name": ns + "-" + suffix, + }, + "snapshotter": snapshotter, + "parameters": parameters, + }, + } + + return snapshotClass +} + // GetUniqueDriverName returns unique driver name that can be used parallelly in tests func GetUniqueDriverName(driver TestDriver) string { return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Config.Framework.UniqueName) diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 442a719f226..4c03b002807 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -23,13 +23,14 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testpatterns" @@ -90,8 +91,10 @@ func createProvisioningTestInput(driver TestDriver, pattern testpatterns.TestPat ExpectedSize: resource.claimSize, }, cs: driver.GetDriverInfo().Config.Framework.ClientSet, + dc: driver.GetDriverInfo().Config.Framework.DynamicClient, pvc: resource.pvc, sc: resource.sc, + vsc: resource.vsc, dInfo: driver.GetDriverInfo(), } @@ -139,6 +142,8 @@ type provisioningTestResource struct { claimSize string sc *storage.StorageClass pvc *v1.PersistentVolumeClaim + // follow parameter is used to test provision volume from snapshot + vsc *unstructured.Unstructured } var _ TestResource = &provisioningTestResource{} @@ -157,6 +162,9 @@ func (p *provisioningTestResource) setupResource(driver TestDriver, pattern test p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name) p.pvc.Spec.StorageClassName = &p.sc.Name framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc) + if sDriver, ok := driver.(SnapshottableTestDriver); ok { + p.vsc = sDriver.GetSnapshotClass() + } } default: framework.Failf("Dynamic Provision test doesn't support: %s", pattern.VolType) @@ -169,8 +177,10 @@ func (p *provisioningTestResource) cleanupResource(driver TestDriver, pattern te type provisioningTestInput struct { testCase StorageClassTest cs clientset.Interface + dc dynamic.Interface pvc *v1.PersistentVolumeClaim sc *storage.StorageClass + vsc *unstructured.Unstructured dInfo *DriverInfo } @@ -198,6 +208,19 @@ func testProvisioning(input *provisioningTestInput) { input.pvc.Spec.VolumeMode = &block TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) }) + + It("should provision storage with snapshot data source [Feature:VolumeSnapshotDataSource]", func() { + if !input.dInfo.Capabilities[CapDataSource] { + framework.Skipf("Driver %q does not support populate data from snapshot - skipping", input.dInfo.Name) + } + + input.testCase.SkipWriteReadCheck = true + dataSource, cleanupFunc := prepareDataSourceForProvisioning(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc) + defer cleanupFunc() + + input.pvc.Spec.DataSource = dataSource + TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) + }) } // TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest and storageClass @@ -205,7 +228,11 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla var err error if class != nil { By("creating a StorageClass " + class.Name) - class, err = client.StorageV1().StorageClasses().Create(class) + _, err = client.StorageV1().StorageClasses().Create(class) + // The "should provision storage with snapshot data source" test already has created the class. + // TODO: make class creation optional and remove the IsAlreadyExists exception + Expect(err == nil || apierrs.IsAlreadyExists(err)).To(Equal(true)) + class, err = client.StorageV1().StorageClasses().Get(class.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) defer func() { framework.Logf("deleting storage class %s", class.Name) @@ -268,6 +295,12 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla Expect(err).NotTo(HaveOccurred()) } + if claim.Spec.DataSource != nil { + By("checking the created volume whether has the pre-populated data") + command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace) + runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, command, t.NodeSelector, t.ExpectUnschedulable) + } + if !t.SkipWriteReadCheck { // We start two pods: // - The first writes 'hello word' to the /mnt/test (= the volume). @@ -288,6 +321,7 @@ func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, cla By("checking the created volume is readable and retains data") runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, "grep 'hello world' /mnt/test/data", t.NodeSelector, t.ExpectUnschedulable) } + By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name)) framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)) @@ -466,3 +500,76 @@ func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeCl Expect(claim.Status.Phase).To(Equal(v1.ClaimPending)) } } + +func prepareDataSourceForProvisioning( + t StorageClassTest, + client clientset.Interface, + dynamicClient dynamic.Interface, + initClaim *v1.PersistentVolumeClaim, + class *storage.StorageClass, + snapshotClass *unstructured.Unstructured, +) (*v1.TypedLocalObjectReference, func()) { + var err error + if class != nil { + By("[Initialize dataSource]creating a StorageClass " + class.Name) + _, err = client.StorageV1().StorageClasses().Create(class) + Expect(err).NotTo(HaveOccurred()) + } + + By("[Initialize dataSource]creating a initClaim") + updatedClaim, err := client.CoreV1().PersistentVolumeClaims(initClaim.Namespace).Create(initClaim) + Expect(err).NotTo(HaveOccurred()) + err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, updatedClaim.Namespace, updatedClaim.Name, framework.Poll, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("[Initialize dataSource]checking the initClaim") + // Get new copy of the initClaim + _, err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Get(updatedClaim.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // write namespace to the /mnt/test (= the volume). + By("[Initialize dataSource]write data to volume") + command := fmt.Sprintf("echo '%s' > /mnt/test/initialData", updatedClaim.GetNamespace()) + runInPodWithVolume(client, updatedClaim.Namespace, updatedClaim.Name, t.NodeName, command, t.NodeSelector, t.ExpectUnschedulable) + + By("[Initialize dataSource]creating a SnapshotClass") + snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{}) + + By("[Initialize dataSource]creating a snapshot") + snapshot := getSnapshot(updatedClaim.Name, updatedClaim.Namespace, snapshotClass.GetName()) + snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(updatedClaim.Namespace).Create(snapshot, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("[Initialize dataSource]checking the snapshot") + // Get new copy of the snapshot + snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + group := "snapshot.storage.k8s.io" + dataSourceRef := &v1.TypedLocalObjectReference{ + APIGroup: &group, + Kind: "VolumeSnapshot", + Name: snapshot.GetName(), + } + + cleanupFunc := func() { + framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName()) + err = dynamicClient.Resource(snapshotGVR).Namespace(updatedClaim.Namespace).Delete(snapshot.GetName(), nil) + if err != nil && !apierrs.IsNotFound(err) { + framework.Failf("Error deleting snapshot %q. Error: %v", snapshot.GetName(), err) + } + + framework.Logf("deleting initClaim %q/%q", updatedClaim.Namespace, updatedClaim.Name) + err = client.CoreV1().PersistentVolumeClaims(updatedClaim.Namespace).Delete(updatedClaim.Name, nil) + if err != nil && !apierrs.IsNotFound(err) { + framework.Failf("Error deleting initClaim %q. Error: %v", updatedClaim.Name, err) + } + + framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName()) + framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil)) + } + + return dataSourceRef, cleanupFunc +} diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go new file mode 100644 index 00000000000..cd3c73a50b8 --- /dev/null +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -0,0 +1,323 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" +) + +// snapshot CRD api group +const snapshotGroup = "snapshot.storage.k8s.io" + +// snapshot CRD api version +const snapshotAPIVersion = "snapshot.storage.k8s.io/v1alpha1" + +var ( + snapshotGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshots"} + snapshotClassGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotclasses"} + snapshotContentGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotcontents"} +) + +type SnapshotClassTest struct { + Name string + CloudProviders []string + Snapshotter string + Parameters map[string]string + NodeName string + NodeSelector map[string]string // NodeSelector for the pod + SnapshotContentCheck func(snapshotContent *unstructured.Unstructured) error +} + +type snapshottableTestSuite struct { + tsInfo TestSuiteInfo +} + +var _ TestSuite = &snapshottableTestSuite{} + +// InitSnapshottableTestSuite returns snapshottableTestSuite that implements TestSuite interface +func InitSnapshottableTestSuite() TestSuite { + return &snapshottableTestSuite{ + tsInfo: TestSuiteInfo{ + name: "snapshottable", + testPatterns: []testpatterns.TestPattern{ + testpatterns.DynamicSnapshot, + }, + }, + } +} + +func (s *snapshottableTestSuite) getTestSuiteInfo() TestSuiteInfo { + return s.tsInfo +} + +func (s *snapshottableTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { +} + +func createSnapshottableTestInput(driver TestDriver, pattern testpatterns.TestPattern) (snapshottableTestResource, snapshottableTestInput) { + // Setup test resource for driver and testpattern + resource := snapshottableTestResource{} + resource.setupResource(driver, pattern) + + input := snapshottableTestInput{ + testCase: SnapshotClassTest{}, + cs: driver.GetDriverInfo().Config.Framework.ClientSet, + dc: driver.GetDriverInfo().Config.Framework.DynamicClient, + pvc: resource.pvc, + sc: resource.sc, + vsc: resource.vsc, + dInfo: driver.GetDriverInfo(), + } + + if driver.GetDriverInfo().Config.ClientNodeName != "" { + input.testCase.NodeName = driver.GetDriverInfo().Config.ClientNodeName + } + + return resource, input +} + +func (s *snapshottableTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { + Context(getTestNameStr(s, pattern), func() { + var ( + resource snapshottableTestResource + input snapshottableTestInput + needsCleanup bool + ) + + BeforeEach(func() { + needsCleanup = false + // Skip unsupported tests to avoid unnecessary resource initialization + skipUnsupportedTest(s, driver, pattern) + needsCleanup = true + + // Create test input + resource, input = createSnapshottableTestInput(driver, pattern) + }) + + AfterEach(func() { + if needsCleanup { + resource.cleanupResource(driver, pattern) + } + }) + + // Ginkgo's "Global Shared Behaviors" require arguments for a shared function + // to be a single struct and to be passed as a pointer. + // Please see https://onsi.github.io/ginkgo/#global-shared-behaviors for details. + testSnapshot(&input) + }) +} + +type snapshottableTestResource struct { + driver TestDriver + claimSize string + + sc *storage.StorageClass + pvc *v1.PersistentVolumeClaim + // volume snapshot class + vsc *unstructured.Unstructured +} + +var _ TestResource = &snapshottableTestResource{} + +func (s *snapshottableTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { + // Setup snapshottableTest resource + switch pattern.SnapshotType { + case testpatterns.DynamicCreatedSnapshot: + if dDriver, ok := driver.(DynamicPVTestDriver); ok { + s.sc = dDriver.GetDynamicProvisionStorageClass("") + if s.sc == nil { + framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name) + } + s.driver = driver + s.claimSize = dDriver.GetClaimSize() + s.pvc = getClaim(s.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name) + s.pvc.Spec.StorageClassName = &s.sc.Name + framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", s.sc, s.pvc) + + if sDriver, ok := driver.(SnapshottableTestDriver); ok { + s.vsc = sDriver.GetSnapshotClass() + } + } + + default: + framework.Failf("Dynamic Snapshot test doesn't support: %s", pattern.SnapshotType) + } +} + +func (s *snapshottableTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { +} + +type snapshottableTestInput struct { + testCase SnapshotClassTest + cs clientset.Interface + dc dynamic.Interface + pvc *v1.PersistentVolumeClaim + sc *storage.StorageClass + // volume snapshot class + vsc *unstructured.Unstructured + dInfo *DriverInfo +} + +func testSnapshot(input *snapshottableTestInput) { + It("should create snapshot with defaults", func() { + if input.dInfo.Name == "csi-hostpath-v0" { + framework.Skipf("skip test when using driver csi-hostpath-v0 - skipping") + } + TestCreateSnapshot(input.testCase, input.cs, input.dc, input.pvc, input.sc, input.vsc) + }) +} + +// TestCreateSnapshot tests dynamic creating snapshot with specified SnapshotClassTest and snapshotClass +func TestCreateSnapshot( + t SnapshotClassTest, + client clientset.Interface, + dynamicClient dynamic.Interface, + claim *v1.PersistentVolumeClaim, + class *storage.StorageClass, + snapshotClass *unstructured.Unstructured, +) *unstructured.Unstructured { + var err error + if class != nil { + By("creating a StorageClass " + class.Name) + class, err = client.StorageV1().StorageClasses().Create(class) + Expect(err).NotTo(HaveOccurred()) + defer func() { + framework.Logf("deleting storage class %s", class.Name) + framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil)) + }() + } + + By("creating a claim") + claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) + Expect(err).NotTo(HaveOccurred()) + defer func() { + framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name) + // typically this claim has already been deleted + err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil) + if err != nil && !apierrs.IsNotFound(err) { + framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err) + } + }() + err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("checking the claim") + // Get new copy of the claim + claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Get the bound PV + pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("creating a SnapshotClass") + snapshotClass, err = dynamicClient.Resource(snapshotClassGVR).Create(snapshotClass, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + framework.Logf("deleting SnapshotClass %s", snapshotClass.GetName()) + framework.ExpectNoError(dynamicClient.Resource(snapshotClassGVR).Delete(snapshotClass.GetName(), nil)) + }() + + By("creating a snapshot") + snapshot := getSnapshot(claim.Name, claim.Namespace, snapshotClass.GetName()) + + snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Create(snapshot, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + framework.Logf("deleting snapshot %q/%q", snapshot.GetNamespace(), snapshot.GetName()) + // typically this snapshot has already been deleted + err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Delete(snapshot.GetName(), nil) + if err != nil && !apierrs.IsNotFound(err) { + framework.Failf("Error deleting snapshot %q. Error: %v", claim.Name, err) + } + }() + err = WaitForSnapshotReady(dynamicClient, snapshot.GetNamespace(), snapshot.GetName(), framework.Poll, framework.SnapshotCreateTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("checking the snapshot") + // Get new copy of the snapshot + snapshot, err = dynamicClient.Resource(snapshotGVR).Namespace(snapshot.GetNamespace()).Get(snapshot.GetName(), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Get the bound snapshotContent + snapshotSpec := snapshot.Object["spec"].(map[string]interface{}) + snapshotContentName := snapshotSpec["snapshotContentName"].(string) + snapshotContent, err := dynamicClient.Resource(snapshotContentGVR).Get(snapshotContentName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + snapshotContentSpec := snapshotContent.Object["spec"].(map[string]interface{}) + volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{}) + persistentVolumeRef := snapshotContentSpec["persistentVolumeRef"].(map[string]interface{}) + + // Check SnapshotContent properties + By("checking the SnapshotContent") + Expect(snapshotContentSpec["snapshotClassName"]).To(Equal(snapshotClass.GetName())) + Expect(volumeSnapshotRef["name"]).To(Equal(snapshot.GetName())) + Expect(volumeSnapshotRef["namespace"]).To(Equal(snapshot.GetNamespace())) + Expect(persistentVolumeRef["name"]).To(Equal(pv.Name)) + + // Run the checker + if t.SnapshotContentCheck != nil { + err = t.SnapshotContentCheck(snapshotContent) + Expect(err).NotTo(HaveOccurred()) + } + + return snapshotContent +} + +// WaitForSnapshotReady waits for a VolumeSnapshot to be ready to use or until timeout occurs, whichever comes first. +func WaitForSnapshotReady(c dynamic.Interface, ns string, snapshotName string, Poll, timeout time.Duration) error { + framework.Logf("Waiting up to %v for VolumeSnapshot %s to become ready", timeout, snapshotName) + for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) { + snapshot, err := c.Resource(snapshotGVR).Namespace(ns).Get(snapshotName, metav1.GetOptions{}) + if err != nil { + framework.Logf("Failed to get claim %q, retrying in %v. Error: %v", snapshotName, Poll, err) + continue + } else { + status := snapshot.Object["status"] + if status == nil { + framework.Logf("VolumeSnapshot %s found but is not ready.", snapshotName) + continue + } + value := status.(map[string]interface{}) + if value["readyToUse"] == true { + framework.Logf("VolumeSnapshot %s found and is ready", snapshotName, time.Since(start)) + return nil + } else if value["ready"] == true { + framework.Logf("VolumeSnapshot %s found and is ready", snapshotName, time.Since(start)) + return nil + } else { + framework.Logf("VolumeSnapshot %s found but is not ready.", snapshotName) + } + } + } + return fmt.Errorf("VolumeSnapshot %s is not ready within %v", snapshotName, timeout) +} diff --git a/test/e2e/storage/testsuites/testdriver.go b/test/e2e/storage/testsuites/testdriver.go index 50badbde15f..7e483dc2443 100644 --- a/test/e2e/storage/testsuites/testdriver.go +++ b/test/e2e/storage/testsuites/testdriver.go @@ -19,6 +19,7 @@ package testsuites import ( "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testpatterns" @@ -79,6 +80,14 @@ type DynamicPVTestDriver interface { GetClaimSize() string } +// SnapshottableTestDriver represents an interface for a TestDriver that supports DynamicSnapshot +type SnapshottableTestDriver interface { + TestDriver + // GetSnapshotClass returns a SnapshotClass to create snapshot. + // It will return nil, if the TestDriver doesn't support it. + GetSnapshotClass() *unstructured.Unstructured +} + // Capability represents a feature that a volume plugin supports type Capability string @@ -87,6 +96,7 @@ const ( CapBlock Capability = "block" // raw block mode CapFsGroup Capability = "fsGroup" // volume ownership via fsGroup CapExec Capability = "exec" // exec a file in the volume + CapDataSource Capability = "dataSource" // support populate data from snapshot ) // DriverInfo represents a combination of parameters to be used in implementation of TestDriver diff --git a/test/e2e/storage/utils/deployment.go b/test/e2e/storage/utils/deployment.go index fa7ddd5995f..e1885eb9e4c 100644 --- a/test/e2e/storage/utils/deployment.go +++ b/test/e2e/storage/utils/deployment.go @@ -93,6 +93,10 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf // Driver name is expected to be the same // as the provisioner here. container.Args = append(container.Args, "--provisioner="+o.NewDriverName) + case o.SnapshotterContainerName: + // Driver name is expected to be the same + // as the snapshotter here. + container.Args = append(container.Args, "--snapshotter="+o.NewDriverName) } } } @@ -145,6 +149,10 @@ type PatchCSIOptions struct { // If non-empty, --provisioner with new name will be appended // to the argument list. ProvisionerContainerName string + // The name of the container which has the snapshotter binary. + // If non-empty, --snapshotter with new name will be appended + // to the argument list. + SnapshotterContainerName string // If non-empty, all pods are forced to run on this node. NodeName string } diff --git a/test/e2e/testing-manifests/storage-csi/controller-role.yaml b/test/e2e/testing-manifests/storage-csi/controller-role.yaml index 4efd6bc98de..4242e420719 100644 --- a/test/e2e/testing-manifests/storage-csi/controller-role.yaml +++ b/test/e2e/testing-manifests/storage-csi/controller-role.yaml @@ -1,3 +1,4 @@ -# Replaced by individual roles for external-attacher and external-provisioner: +# Replaced by individual roles for external-attacher, external-provisioner and external-snapshotter: # - https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml # - https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml +# - https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml \ No newline at end of file diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md b/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md new file mode 100644 index 00000000000..90dab9588d8 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md @@ -0,0 +1 @@ +The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml new file mode 100644 index 00000000000..f29db319289 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml @@ -0,0 +1,65 @@ +# Together with the RBAC file for external-provisioner, this YAML file +# contains all RBAC objects that are necessary to run external CSI +# snapshotter. +# +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole +# - optionally rename the non-namespaced ClusterRole if there +# are conflicts with other deployments + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-snapshotter + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: external-snapshotter-runner +rules: +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-role +subjects: +- kind: ServiceAccount + name: csi-snapshotter + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + # change the name also here if the ClusterRole gets renamed + name: external-snapshotter-runner + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/README.md b/test/e2e/testing-manifests/storage-csi/hostpath/README.md index 6a46c1177df..c2990622700 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/README.md +++ b/test/e2e/testing-manifests/storage-csi/hostpath/README.md @@ -1,5 +1,5 @@ A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example, with some modifications: - serviceAccountName is used instead of the deprecated serviceAccount -- the RBAC roles from driver-registrar, external-attacher and external-provisioner - are used +- the RBAC roles from driver-registrar, external-attacher, external-provisioner + and external-snapshotter are used diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml new file mode 100644 index 00000000000..5fa5011e67a --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml @@ -0,0 +1,48 @@ +kind: Service +apiVersion: v1 +metadata: + name: csi-snapshotter + labels: + app: csi-snapshotter +spec: + selector: + app: csi-snapshotter + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-snapshotter +spec: + serviceName: "csi-snapshotter" + replicas: 1 + selector: + matchLabels: + app: csi-snapshotter + template: + metadata: + labels: + app: csi-snapshotter + spec: + serviceAccount: csi-snapshotter + containers: + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v1.0.1 + args: + - "--csi-address=$(ADDRESS)" + - "--connection-timeout=15s" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - name: socket-dir + mountPath: /csi + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir \ No newline at end of file diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml index aa008ecac29..c0ec6c0378d 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml @@ -13,6 +13,9 @@ subjects: - kind: ServiceAccount name: csi-provisioner namespace: default + - kind: ServiceAccount + name: csi-snapshotter + namespace: default roleRef: kind: ClusterRole name: e2e-test-privileged-psp