diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index c0919e6c5dd..9db126fdd1c 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -31,6 +31,7 @@ import ( csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -49,7 +50,7 @@ const ( type csiTestDriver interface { createCSIDriver() cleanupCSIDriver() - createStorageClassTest(node v1.Node) storageClassTest + createStorageClassTest(node v1.Node) testsuites.StorageClassTest } var csiTestDrivers = map[string]func(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver{ @@ -110,7 +111,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { claim := newClaim(t, ns.GetName(), "") class := newStorageClass(t, ns.GetName(), "") claim.Spec.StorageClassName = &class.ObjectMeta.Name - testDynamicProvisioning(t, cs, claim, class) + testsuites.TestDynamicProvisioning(t, cs, claim, class) }) }) } @@ -187,7 +188,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { By("Checking if VolumeAttachment was created for the pod") // Check that VolumeAttachment does not exist handle := getVolumeHandle(cs, claim) - attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, t.provisioner, node.Name))) + attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, t.Provisioner, node.Name))) attachmentName := fmt.Sprintf("csi-%x", attachmentHash) _, err = cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{}) if err != nil { @@ -242,7 +243,7 @@ func getVolumeHandle(cs clientset.Interface, claim *v1.PersistentVolumeClaim) st return pv.Spec.CSI.VolumeHandle } -func startPausePod(cs clientset.Interface, t storageClassTest, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { +func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { class := newStorageClass(t, ns, "") class, err := cs.StorageV1().StorageClasses().Create(class) framework.ExpectNoError(err, "Failed to create class : %v", err) @@ -283,8 +284,8 @@ func startPausePod(cs clientset.Interface, t storageClassTest, ns string) (*stor }, } - if len(t.nodeName) != 0 { - pod.Spec.NodeName = t.nodeName + if len(t.NodeName) != 0 { + pod.Spec.NodeName = t.NodeName } pod, err = cs.CoreV1().Pods(ns).Create(pod) framework.ExpectNoError(err, "Failed to create pod: %v", err) @@ -311,14 +312,14 @@ func initCSIHostpath(f *framework.Framework, config framework.VolumeTestConfig) } } -func (h *hostpathCSIDriver) createStorageClassTest(node v1.Node) storageClassTest { - return storageClassTest{ - name: "csi-hostpath", - provisioner: "csi-hostpath", - parameters: map[string]string{}, - claimSize: "1Gi", - expectedSize: "1Gi", - nodeName: node.Name, +func (h *hostpathCSIDriver) createStorageClassTest(node v1.Node) testsuites.StorageClassTest { + return testsuites.StorageClassTest{ + Name: "csi-hostpath", + Provisioner: "csi-hostpath", + Parameters: map[string]string{}, + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + NodeName: node.Name, } } @@ -379,14 +380,14 @@ func initCSIgcePD(f *framework.Framework, config framework.VolumeTestConfig) csi } } -func (g *gcePDCSIDriver) createStorageClassTest(node v1.Node) storageClassTest { - return storageClassTest{ - name: "com.google.csi.gcepd", - provisioner: "com.google.csi.gcepd", - parameters: map[string]string{"type": "pd-standard"}, - claimSize: "5Gi", - expectedSize: "5Gi", - nodeName: node.Name, +func (g *gcePDCSIDriver) createStorageClassTest(node v1.Node) testsuites.StorageClassTest { + return testsuites.StorageClassTest{ + Name: "com.google.csi.gcepd", + Provisioner: "com.google.csi.gcepd", + Parameters: map[string]string{"type": "pd-standard"}, + ClaimSize: "5Gi", + ExpectedSize: "5Gi", + NodeName: node.Name, } } diff --git a/test/e2e/storage/drivers/base.go b/test/e2e/storage/drivers/base.go index e6d4109a9c6..5d5ca0e1363 100644 --- a/test/e2e/storage/drivers/base.go +++ b/test/e2e/storage/drivers/base.go @@ -81,11 +81,13 @@ type DriverInfo struct { Name string // Name of the driver FeatureTag string // FeatureTag for the driver - MaxFileSize int64 // Max file size to be tested for this driver - SupportedFsType sets.String // Map of string for supported fs type - IsPersistent bool // Flag to represent whether it provides persistency - IsFsGroupSupported bool // Flag to represent whether it supports fsGroup - IsBlockSupported bool // Flag to represent whether it supports Block Volume + MaxFileSize int64 // Max file size to be tested for this driver + SupportedFsType sets.String // Map of string for supported fs type + SupportedMountOption sets.String // Map of string for supported mount option + RequiredMountOption sets.String // Map of string for required mount option (Optional) + IsPersistent bool // Flag to represent whether it provides persistency + IsFsGroupSupported bool // Flag to represent whether it supports fsGroup + IsBlockSupported bool // Flag to represent whether it supports Block Volume // Parameters below will be set inside test loop by using SetCommonDriverParameters. // Drivers that implement TestDriver is required to set all the above parameters @@ -104,8 +106,8 @@ func GetDriverNameWithFeatureTags(driver TestDriver) string { return fmt.Sprintf("[Driver: %s]%s", dInfo.Name, dInfo.FeatureTag) } +// CreateVolume creates volume for test unless dynamicPV test func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface{} { - // Create Volume for test unless dynamicPV test switch volType { case testpatterns.InlineVolume: fallthrough @@ -121,8 +123,8 @@ func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface return nil } +// DeleteVolume deletes volume for test unless dynamicPV test func DeleteVolume(driver TestDriver, volType testpatterns.TestVolType, testResource interface{}) { - // Delete Volume for test unless dynamicPV test switch volType { case testpatterns.InlineVolume: fallthrough diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index f2f8681e482..da0da68039d 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -88,9 +88,11 @@ func InitNFSDriver() TestDriver { SupportedFsType: sets.NewString( "", // Default fsType ), - IsPersistent: true, - IsFsGroupSupported: false, - IsBlockSupported: false, + SupportedMountOption: sets.NewString("proto=tcp", "nosuid"), + RequiredMountOption: sets.NewString("vers=4.1"), + IsPersistent: true, + IsFsGroupSupported: false, + IsBlockSupported: false, }, } } @@ -673,7 +675,7 @@ var _ TestDriver = &hostPathDriver{} var _ PreprovisionedVolumeTestDriver = &hostPathDriver{} var _ InlineVolumeTestDriver = &hostPathDriver{} -// InitHostpathDriver returns hostPathDriver that implements TestDriver interface +// InitHostPathDriver returns hostPathDriver that implements TestDriver interface func InitHostPathDriver() TestDriver { return &hostPathDriver{ driverInfo: DriverInfo{ @@ -1118,9 +1120,10 @@ func InitGcePdDriver() TestDriver { "ext4", "xfs", ), - IsPersistent: true, - IsFsGroupSupported: true, - IsBlockSupported: true, + SupportedMountOption: sets.NewString("debug", "nouid32"), + IsPersistent: true, + IsFsGroupSupported: true, + IsBlockSupported: true, }, } } @@ -1460,9 +1463,10 @@ func InitAwsDriver() TestDriver { "", // Default fsType "ext3", ), - IsPersistent: true, - IsFsGroupSupported: true, - IsBlockSupported: true, + SupportedMountOption: sets.NewString("debug", "nouid32"), + IsPersistent: true, + IsFsGroupSupported: true, + IsBlockSupported: true, }, } } diff --git a/test/e2e/storage/generic_persistent_volume-disruptive.go b/test/e2e/storage/generic_persistent_volume-disruptive.go index fdba7e78433..8b428f6f204 100644 --- a/test/e2e/storage/generic_persistent_volume-disruptive.go +++ b/test/e2e/storage/generic_persistent_volume-disruptive.go @@ -23,6 +23,7 @@ import ( "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -85,9 +86,9 @@ var _ = utils.SIGDescribe("GenericPersistentVolume[Disruptive]", func() { func createPodPVCFromSC(f *framework.Framework, c clientset.Interface, ns string) (*v1.Pod, *v1.PersistentVolumeClaim, *v1.PersistentVolume) { var err error - test := storageClassTest{ - name: "default", - claimSize: "2Gi", + test := testsuites.StorageClassTest{ + Name: "default", + ClaimSize: "2Gi", } pvc := newClaim(test, ns, "default") pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 86f5b1be6dc..2906a17cecc 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -50,6 +50,7 @@ var testSuites = []func() testsuites.TestSuite{ testsuites.InitVolumeIOTestSuite, testsuites.InitVolumeModeTestSuite, testsuites.InitSubPathTestSuite, + testsuites.InitProvisioningTestSuite, } // This executes testSuites for in-tree volumes. diff --git a/test/e2e/storage/mounted_volume_resize.go b/test/e2e/storage/mounted_volume_resize.go index d7eedfc8fe4..54feb244aac 100644 --- a/test/e2e/storage/mounted_volume_resize.go +++ b/test/e2e/storage/mounted_volume_resize.go @@ -31,6 +31,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/client/conditions" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -72,9 +73,9 @@ var _ = utils.SIGDescribe("Mounted volume expand[Slow]", func() { isNodeLabeled = true } - test := storageClassTest{ - name: "default", - claimSize: "2Gi", + test := testsuites.StorageClassTest{ + Name: "default", + ClaimSize: "2Gi", } resizableSc, err = createResizableStorageClass(test, ns, "resizing", c) Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class") diff --git a/test/e2e/storage/pvc_protection.go b/test/e2e/storage/pvc_protection.go index 2d2346ee134..af6ed9c1281 100644 --- a/test/e2e/storage/pvc_protection.go +++ b/test/e2e/storage/pvc_protection.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/pkg/util/slice" volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -47,8 +48,8 @@ var _ = utils.SIGDescribe("PVC Protection", func() { By("Creating a PVC") suffix := "pvc-protection" defaultSC := getDefaultStorageClassName(client) - testStorageClass := storageClassTest{ - claimSize: "1Gi", + testStorageClass := testsuites.StorageClassTest{ + ClaimSize: "1Gi", } pvc = newClaim(testStorageClass, nameSpace, suffix) pvc.Spec.StorageClassName = &defaultSC diff --git a/test/e2e/storage/regional_pd.go b/test/e2e/storage/regional_pd.go index 4fc0cfd24f9..4c3a517721d 100644 --- a/test/e2e/storage/regional_pd.go +++ b/test/e2e/storage/regional_pd.go @@ -38,6 +38,7 @@ import ( "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/providers/gce" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -90,19 +91,19 @@ func testVolumeProvisioning(c clientset.Interface, ns string) { // This test checks that dynamic provisioning can provision a volume // that can be used to persist data among pods. - tests := []storageClassTest{ + tests := []testsuites.StorageClassTest{ { - name: "HDD Regional PD on GCE/GKE", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ + Name: "HDD Regional PD on GCE/GKE", + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + Parameters: map[string]string{ "type": "pd-standard", "zones": strings.Join(cloudZones, ","), "replication-type": "regional-pd", }, - claimSize: "1.5Gi", - expectedSize: "2Gi", - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "1.5Gi", + ExpectedSize: "2Gi", + PvCheck: func(volume *v1.PersistentVolume) error { err := checkGCEPD(volume, "pd-standard") if err != nil { return err @@ -111,16 +112,16 @@ func testVolumeProvisioning(c clientset.Interface, ns string) { }, }, { - name: "HDD Regional PD with auto zone selection on GCE/GKE", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ + Name: "HDD Regional PD with auto zone selection on GCE/GKE", + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + Parameters: map[string]string{ "type": "pd-standard", "replication-type": "regional-pd", }, - claimSize: "1.5Gi", - expectedSize: "2Gi", - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "1.5Gi", + ExpectedSize: "2Gi", + PvCheck: func(volume *v1.PersistentVolume) error { err := checkGCEPD(volume, "pd-standard") if err != nil { return err @@ -138,7 +139,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) { class := newStorageClass(test, ns, "" /* suffix */) claim := newClaim(test, ns, "" /* suffix */) claim.Spec.StorageClassName = &class.Name - testDynamicProvisioning(test, c, claim, class) + testsuites.TestDynamicProvisioning(test, c, claim, class) } } @@ -278,15 +279,15 @@ func testZonalFailover(c clientset.Interface, ns string) { } func testRegionalDelayedBinding(c clientset.Interface, ns string) { - test := storageClassTest{ - name: "Regional PD storage class with waitForFirstConsumer test on GCE", - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ + test := testsuites.StorageClassTest{ + Name: "Regional PD storage class with waitForFirstConsumer test on GCE", + Provisioner: "kubernetes.io/gce-pd", + Parameters: map[string]string{ "type": "pd-standard", "replication-type": "regional-pd", }, - claimSize: "2Gi", - delayBinding: true, + ClaimSize: "2Gi", + DelayBinding: true, } suffix := "delayed-regional" @@ -305,15 +306,15 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) { } func testRegionalAllowedTopologies(c clientset.Interface, ns string) { - test := storageClassTest{ - name: "Regional PD storage class with allowedTopologies test on GCE", - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ + test := testsuites.StorageClassTest{ + Name: "Regional PD storage class with allowedTopologies test on GCE", + Provisioner: "kubernetes.io/gce-pd", + Parameters: map[string]string{ "type": "pd-standard", "replication-type": "regional-pd", }, - claimSize: "2Gi", - expectedSize: "2Gi", + ClaimSize: "2Gi", + ExpectedSize: "2Gi", } suffix := "topo-regional" @@ -322,20 +323,20 @@ func testRegionalAllowedTopologies(c clientset.Interface, ns string) { addAllowedTopologiesToStorageClass(c, class, zones) claim := newClaim(test, ns, suffix) claim.Spec.StorageClassName = &class.Name - pv := testDynamicProvisioning(test, c, claim, class) + pv := testsuites.TestDynamicProvisioning(test, c, claim, class) checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true) } func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string) { - test := storageClassTest{ - name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE", - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ + test := testsuites.StorageClassTest{ + Name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE", + Provisioner: "kubernetes.io/gce-pd", + Parameters: map[string]string{ "type": "pd-standard", "replication-type": "regional-pd", }, - claimSize: "2Gi", - delayBinding: true, + ClaimSize: "2Gi", + DelayBinding: true, } suffix := "topo-delayed-regional" diff --git a/test/e2e/storage/testsuites/BUILD b/test/e2e/storage/testsuites/BUILD index e32594b8714..1987fb2cf50 100644 --- a/test/e2e/storage/testsuites/BUILD +++ b/test/e2e/storage/testsuites/BUILD @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "base.go", + "provisioning.go", "subpath.go", "volume_io.go", "volumemode.go", diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index dd3e482daf3..fb8bde814f5 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -34,7 +34,7 @@ import ( "k8s.io/kubernetes/test/e2e/storage/testpatterns" ) -// TestSuite represents an interface for a set of tests whchi works with TestDriver +// TestSuite represents an interface for a set of tests which works with TestDriver type TestSuite interface { // getTestSuiteInfo returns the TestSuiteInfo for this TestSuite getTestSuiteInfo() TestSuiteInfo @@ -44,6 +44,7 @@ type TestSuite interface { execTest(drivers.TestDriver, testpatterns.TestPattern) } +// TestSuiteInfo represents a set of parameters for TestSuite type TestSuiteInfo struct { name string // name of the TestSuite featureTag string // featureTag for the TestSuite @@ -132,7 +133,7 @@ type genericVolumeTestResource struct { var _ TestResource = &genericVolumeTestResource{} -// SetupResource sets up genericVolumeTestResource +// setupResource sets up genericVolumeTestResource func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { r.driver = driver dInfo := driver.GetDriverInfo() @@ -186,7 +187,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat } } -// CleanupResource clean up genericVolumeTestResource +// cleanupResource cleans up genericVolumeTestResource func (r *genericVolumeTestResource) cleanupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() f := dInfo.Framework diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go new file mode 100644 index 00000000000..2047b4d846a --- /dev/null +++ b/test/e2e/storage/testsuites/provisioning.go @@ -0,0 +1,367 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + + "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/drivers" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" + imageutils "k8s.io/kubernetes/test/utils/image" +) + +// StorageClassTest represents parameters to be used by provisioning tests +type StorageClassTest struct { + Name string + CloudProviders []string + Provisioner string + Parameters map[string]string + DelayBinding bool + ClaimSize string + ExpectedSize string + PvCheck func(volume *v1.PersistentVolume) error + NodeName string + SkipWriteReadCheck bool + VolumeMode *v1.PersistentVolumeMode +} + +type provisioningTestSuite struct { + tsInfo TestSuiteInfo +} + +var _ TestSuite = &provisioningTestSuite{} + +// InitProvisioningTestSuite returns provisioningTestSuite that implements TestSuite interface +func InitProvisioningTestSuite() TestSuite { + return &provisioningTestSuite{ + tsInfo: TestSuiteInfo{ + name: "provisioning", + testPatterns: []testpatterns.TestPattern{ + testpatterns.DefaultFsDynamicPV, + }, + }, + } +} + +func (p *provisioningTestSuite) getTestSuiteInfo() TestSuiteInfo { + return p.tsInfo +} + +func (p *provisioningTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver drivers.TestDriver) { +} + +func createProvisioningTestInput(driver drivers.TestDriver, pattern testpatterns.TestPattern) (provisioningTestResource, provisioningTestInput) { + // Setup test resource for driver and testpattern + resource := provisioningTestResource{} + resource.setupResource(driver, pattern) + + input := provisioningTestInput{ + testCase: StorageClassTest{ + ClaimSize: resource.claimSize, + ExpectedSize: resource.claimSize, + }, + cs: driver.GetDriverInfo().Framework.ClientSet, + pvc: resource.pvc, + sc: resource.sc, + dInfo: driver.GetDriverInfo(), + } + + if driver.GetDriverInfo().Config.ClientNodeName != "" { + input.testCase.NodeName = driver.GetDriverInfo().Config.ClientNodeName + } + + return resource, input +} + +func (p *provisioningTestSuite) execTest(driver drivers.TestDriver, pattern testpatterns.TestPattern) { + Context(getTestNameStr(p, pattern), func() { + var ( + resource provisioningTestResource + input provisioningTestInput + needsCleanup bool + ) + + BeforeEach(func() { + needsCleanup = false + // Skip unsupported tests to avoid unnecessary resource initialization + skipUnsupportedTest(p, driver, pattern) + needsCleanup = true + + // Create test input + resource, input = createProvisioningTestInput(driver, pattern) + }) + + AfterEach(func() { + if needsCleanup { + resource.cleanupResource(driver, pattern) + } + }) + + // Ginkgo's "Global Shared Behaviors" require arguments for a shared function + // to be a single struct and to be passed as a pointer. + // Please see https://onsi.github.io/ginkgo/#global-shared-behaviors for details. + testProvisioning(&input) + }) +} + +type provisioningTestResource struct { + driver drivers.TestDriver + + claimSize string + sc *storage.StorageClass + pvc *v1.PersistentVolumeClaim +} + +var _ TestResource = &provisioningTestResource{} + +func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { + // Setup provisioningTest resource + switch pattern.VolType { + case testpatterns.DynamicPV: + if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { + p.sc = dDriver.GetDynamicProvisionStorageClass("") + if p.sc == nil { + framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name) + } + p.driver = driver + p.claimSize = "2Gi" + p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name) + p.pvc.Spec.StorageClassName = &p.sc.Name + framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc) + } + default: + framework.Failf("Dynamic Provision test doesn't support: %s", pattern.VolType) + } +} + +func (p *provisioningTestResource) cleanupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +} + +type provisioningTestInput struct { + testCase StorageClassTest + cs clientset.Interface + pvc *v1.PersistentVolumeClaim + sc *storage.StorageClass + dInfo *drivers.DriverInfo +} + +func testProvisioning(input *provisioningTestInput) { + It("should provision storage", func() { + TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) + }) + + It("should provision storage with mount options", func() { + if input.dInfo.SupportedMountOption == nil { + framework.Skipf("Driver %q does not define supported mount option - skipping", input.dInfo.Name) + } + + input.sc.MountOptions = input.dInfo.SupportedMountOption.Union(input.dInfo.RequiredMountOption).List() + TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) + }) + + It("should provision storage with non-default reclaim policy Retain", func() { + retain := v1.PersistentVolumeReclaimRetain + input.sc.ReclaimPolicy = &retain + pv := TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) + + By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased)) + framework.ExpectNoError(framework.WaitForPersistentVolumePhase(v1.VolumeReleased, input.cs, pv.Name, 1*time.Second, 30*time.Second)) + + By(fmt.Sprintf("deleting the PV %q", pv.Name)) + framework.ExpectNoError(framework.DeletePersistentVolume(input.cs, pv.Name), "Failed to delete PV ", pv.Name) + framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(input.cs, pv.Name, 1*time.Second, 30*time.Second)) + }) + + It("should create and delete block persistent volumes [Feature:BlockVolume]", func() { + if !input.dInfo.IsBlockSupported { + framework.Skipf("Driver %q does not support BlockVolume - skipping", input.dInfo.Name) + } + block := v1.PersistentVolumeBlock + input.testCase.VolumeMode = &block + input.testCase.SkipWriteReadCheck = true + input.pvc.Spec.VolumeMode = &block + TestDynamicProvisioning(input.testCase, input.cs, input.pvc, input.sc) + }) +} + +// TestDynamicProvisioning tests dynamic provisioning with specified StorageClassTest and storageClass +func TestDynamicProvisioning(t StorageClassTest, client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) *v1.PersistentVolume { + var err error + if class != nil { + By("creating a StorageClass " + class.Name) + class, err = client.StorageV1().StorageClasses().Create(class) + Expect(err).NotTo(HaveOccurred()) + defer func() { + framework.Logf("deleting storage class %s", class.Name) + framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil)) + }() + } + + By("creating a claim") + claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) + Expect(err).NotTo(HaveOccurred()) + defer func() { + framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name) + // typically this claim has already been deleted + err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil) + if err != nil && !apierrs.IsNotFound(err) { + framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err) + } + }() + err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("checking the claim") + // Get new copy of the claim + claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Get the bound PV + pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Check sizes + expectedCapacity := resource.MustParse(t.ExpectedSize) + pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)] + Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity") + + requestedCapacity := resource.MustParse(t.ClaimSize) + claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] + Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity") + + // Check PV properties + By("checking the PV") + expectedAccessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} + Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes)) + Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name)) + Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace)) + if class == nil { + Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete)) + } else { + Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(*class.ReclaimPolicy)) + Expect(pv.Spec.MountOptions).To(Equal(class.MountOptions)) + } + if t.VolumeMode != nil { + Expect(pv.Spec.VolumeMode).NotTo(BeNil()) + Expect(*pv.Spec.VolumeMode).To(Equal(*t.VolumeMode)) + } + + // Run the checker + if t.PvCheck != nil { + err = t.PvCheck(pv) + Expect(err).NotTo(HaveOccurred()) + } + + if !t.SkipWriteReadCheck { + // We start two pods: + // - The first writes 'hello word' to the /mnt/test (= the volume). + // - The second one runs grep 'hello world' on /mnt/test. + // If both succeed, Kubernetes actually allocated something that is + // persistent across pods. + By("checking the created volume is writable and has the PV's mount options") + command := "echo 'hello world' > /mnt/test/data" + // We give the first pod the secondary responsibility of checking the volume has + // been mounted with the PV's mount options, if the PV was provisioned with any + for _, option := range pv.Spec.MountOptions { + // Get entry, get mount options at 6th word, replace brackets with commas + command += fmt.Sprintf(" && ( mount | grep 'on /mnt/test' | awk '{print $6}' | sed 's/^(/,/; s/)$/,/' | grep -q ,%s, )", option) + } + runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, command) + + By("checking the created volume is readable and retains data") + runInPodWithVolume(client, claim.Namespace, claim.Name, t.NodeName, "grep 'hello world' /mnt/test/data") + } + By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name)) + framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)) + + // Wait for the PV to get deleted if reclaim policy is Delete. (If it's + // Retain, there's no use waiting because the PV won't be auto-deleted and + // it's expected for the caller to do it.) Technically, the first few delete + // attempts may fail, as the volume is still attached to a node because + // kubelet is slowly cleaning up the previous pod, however it should succeed + // in a couple of minutes. Wait 20 minutes to recover from random cloud + // hiccups. + if pv.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete { + By(fmt.Sprintf("deleting the claim's PV %q", pv.Name)) + framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute)) + } + + return pv +} + +// runInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory. +func runInPodWithVolume(c clientset.Interface, ns, claimName, nodeName, command string) { + pod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-volume-tester-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "volume-tester", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"/bin/sh"}, + Args: []string{"-c", command}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "my-volume", + MountPath: "/mnt/test", + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "my-volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: claimName, + ReadOnly: false, + }, + }, + }, + }, + }, + } + + if len(nodeName) != 0 { + pod.Spec.NodeName = nodeName + } + pod, err := c.CoreV1().Pods(ns).Create(pod) + framework.ExpectNoError(err, "Failed to create pod: %v", err) + defer func() { + framework.DeletePodOrFail(c, ns, pod.Name) + }() + framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace)) +} diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index c6cd4e8a3b0..7772132362a 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -338,7 +338,7 @@ func StartExternalProvisioner(c clientset.Interface, ns string, externalPluginNa Containers: []v1.Container{ { Name: "nfs-provisioner", - Image: "quay.io/kubernetes_incubator/nfs-provisioner:v2.1.0-k8s1.11", + Image: "quay.io/kubernetes_incubator/nfs-provisioner:v2.2.0-k8s1.12", SecurityContext: &v1.SecurityContext{ Capabilities: &v1.Capabilities{ Add: []v1.Capability{"DAC_READ_SEARCH"}, diff --git a/test/e2e/storage/volume_expand.go b/test/e2e/storage/volume_expand.go index 8049d1eda0c..1927e970adc 100644 --- a/test/e2e/storage/volume_expand.go +++ b/test/e2e/storage/volume_expand.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -54,9 +55,9 @@ var _ = utils.SIGDescribe("Volume expand [Slow]", func() { c = f.ClientSet ns = f.Namespace.Name framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) - test := storageClassTest{ - name: "default", - claimSize: "2Gi", + test := testsuites.StorageClassTest{ + Name: "default", + ClaimSize: "2Gi", } resizableSc, err = createResizableStorageClass(test, ns, "resizing", c) Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class") @@ -133,7 +134,7 @@ var _ = utils.SIGDescribe("Volume expand [Slow]", func() { }) }) -func createResizableStorageClass(t storageClassTest, ns string, suffix string, c clientset.Interface) (*storage.StorageClass, error) { +func createResizableStorageClass(t testsuites.StorageClassTest, ns string, suffix string, c clientset.Interface) (*storage.StorageClass, error) { stKlass := newStorageClass(t, ns, suffix) allowExpansion := true stKlass.AllowVolumeExpansion = &allowExpansion diff --git a/test/e2e/storage/volume_metrics.go b/test/e2e/storage/volume_metrics.go index f6e5d130dd5..8c1dccc2d05 100644 --- a/test/e2e/storage/volume_metrics.go +++ b/test/e2e/storage/volume_metrics.go @@ -31,6 +31,7 @@ import ( kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/metrics" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -52,9 +53,9 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() { defaultScName := getDefaultStorageClassName(c) verifyDefaultStorageClass(c, defaultScName, true) - test := storageClassTest{ - name: "default", - claimSize: "2Gi", + test := testsuites.StorageClassTest{ + Name: "default", + ClaimSize: "2Gi", } pvc = newClaim(test, ns, "default") diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go index 6feaed756b5..53079e5ea7f 100644 --- a/test/e2e/storage/volume_provisioning.go +++ b/test/e2e/storage/volume_provisioning.go @@ -47,133 +47,15 @@ import ( volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/providers/gce" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" - imageutils "k8s.io/kubernetes/test/utils/image" ) -type storageClassTest struct { - name string - cloudProviders []string - provisioner string - parameters map[string]string - delayBinding bool - claimSize string - expectedSize string - pvCheck func(volume *v1.PersistentVolume) error - nodeName string - skipWriteReadCheck bool - volumeMode *v1.PersistentVolumeMode -} - const ( // Plugin name of the external provisioner externalPluginName = "example.com/nfs" ) -func testDynamicProvisioning(t storageClassTest, client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) *v1.PersistentVolume { - var err error - if class != nil { - By("creating a StorageClass " + class.Name) - class, err = client.StorageV1().StorageClasses().Create(class) - Expect(err).NotTo(HaveOccurred()) - defer func() { - framework.Logf("deleting storage class %s", class.Name) - framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(class.Name, nil)) - }() - } - - By("creating a claim") - claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) - Expect(err).NotTo(HaveOccurred()) - defer func() { - framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name) - // typically this claim has already been deleted - err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil) - if err != nil && !apierrs.IsNotFound(err) { - framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err) - } - }() - err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) - Expect(err).NotTo(HaveOccurred()) - - By("checking the claim") - // Get new copy of the claim - claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - - // Get the bound PV - pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - - // Check sizes - expectedCapacity := resource.MustParse(t.expectedSize) - pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)] - Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity") - - requestedCapacity := resource.MustParse(t.claimSize) - claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] - Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity") - - // Check PV properties - By("checking the PV") - expectedAccessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} - Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes)) - Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name)) - Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace)) - if class == nil { - Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete)) - } else { - Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(*class.ReclaimPolicy)) - Expect(pv.Spec.MountOptions).To(Equal(class.MountOptions)) - } - if t.volumeMode != nil { - Expect(pv.Spec.VolumeMode).NotTo(BeNil()) - Expect(*pv.Spec.VolumeMode).To(Equal(*t.volumeMode)) - } - - // Run the checker - if t.pvCheck != nil { - err = t.pvCheck(pv) - Expect(err).NotTo(HaveOccurred()) - } - - if !t.skipWriteReadCheck { - // We start two pods: - // - The first writes 'hello word' to the /mnt/test (= the volume). - // - The second one runs grep 'hello world' on /mnt/test. - // If both succeed, Kubernetes actually allocated something that is - // persistent across pods. - By("checking the created volume is writable and has the PV's mount options") - command := "echo 'hello world' > /mnt/test/data" - // We give the first pod the secondary responsibility of checking the volume has - // been mounted with the PV's mount options, if the PV was provisioned with any - for _, option := range pv.Spec.MountOptions { - // Get entry, get mount options at 6th word, replace brackets with commas - command += fmt.Sprintf(" && ( mount | grep 'on /mnt/test' | awk '{print $6}' | sed 's/^(/,/; s/)$/,/' | grep -q ,%s, )", option) - } - runInPodWithVolume(client, claim.Namespace, claim.Name, t.nodeName, command) - - By("checking the created volume is readable and retains data") - runInPodWithVolume(client, claim.Namespace, claim.Name, t.nodeName, "grep 'hello world' /mnt/test/data") - } - By(fmt.Sprintf("deleting claim %q/%q", claim.Namespace, claim.Name)) - framework.ExpectNoError(client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)) - - // Wait for the PV to get deleted if reclaim policy is Delete. (If it's - // Retain, there's no use waiting because the PV won't be auto-deleted and - // it's expected for the caller to do it.) Technically, the first few delete - // attempts may fail, as the volume is still attached to a node because - // kubelet is slowly cleaning up the previous pod, however it should succeed - // in a couple of minutes. Wait 20 minutes to recover from random cloud - // hiccups. - if pv.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete { - By(fmt.Sprintf("deleting the claim's PV %q", pv.Name)) - framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute)) - } - - return pv -} - func testBindingWaitForFirstConsumer(client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) (*v1.PersistentVolume, *v1.Node) { var err error @@ -367,172 +249,172 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { // This test checks that dynamic provisioning can provision a volume // that can be used to persist data among pods. - tests := []storageClassTest{ + tests := []testsuites.StorageClassTest{ // GCE/GKE { - name: "SSD PD on GCE/GKE", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ + Name: "SSD PD on GCE/GKE", + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + Parameters: map[string]string{ "type": "pd-ssd", "zone": cloudZone, }, - claimSize: "1.5Gi", - expectedSize: "2Gi", - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "1.5Gi", + ExpectedSize: "2Gi", + PvCheck: func(volume *v1.PersistentVolume) error { return checkGCEPD(volume, "pd-ssd") }, }, { - name: "HDD PD on GCE/GKE", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ + Name: "HDD PD on GCE/GKE", + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + Parameters: map[string]string{ "type": "pd-standard", }, - claimSize: "1.5Gi", - expectedSize: "2Gi", - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "1.5Gi", + ExpectedSize: "2Gi", + PvCheck: func(volume *v1.PersistentVolume) error { return checkGCEPD(volume, "pd-standard") }, }, // AWS { - name: "gp2 EBS on AWS", - cloudProviders: []string{"aws"}, - provisioner: "kubernetes.io/aws-ebs", - parameters: map[string]string{ + Name: "gp2 EBS on AWS", + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + Parameters: map[string]string{ "type": "gp2", "zone": cloudZone, }, - claimSize: "1.5Gi", - expectedSize: "2Gi", - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "1.5Gi", + ExpectedSize: "2Gi", + PvCheck: func(volume *v1.PersistentVolume) error { return checkAWSEBS(volume, "gp2", false) }, }, { - name: "io1 EBS on AWS", - cloudProviders: []string{"aws"}, - provisioner: "kubernetes.io/aws-ebs", - parameters: map[string]string{ + Name: "io1 EBS on AWS", + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + Parameters: map[string]string{ "type": "io1", "iopsPerGB": "50", }, - claimSize: "3.5Gi", - expectedSize: "4Gi", // 4 GiB is minimum for io1 - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "3.5Gi", + ExpectedSize: "4Gi", // 4 GiB is minimum for io1 + PvCheck: func(volume *v1.PersistentVolume) error { return checkAWSEBS(volume, "io1", false) }, }, { - name: "sc1 EBS on AWS", - cloudProviders: []string{"aws"}, - provisioner: "kubernetes.io/aws-ebs", - parameters: map[string]string{ + Name: "sc1 EBS on AWS", + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + Parameters: map[string]string{ "type": "sc1", }, - claimSize: "500Gi", // minimum for sc1 - expectedSize: "500Gi", - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "500Gi", // minimum for sc1 + ExpectedSize: "500Gi", + PvCheck: func(volume *v1.PersistentVolume) error { return checkAWSEBS(volume, "sc1", false) }, }, { - name: "st1 EBS on AWS", - cloudProviders: []string{"aws"}, - provisioner: "kubernetes.io/aws-ebs", - parameters: map[string]string{ + Name: "st1 EBS on AWS", + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + Parameters: map[string]string{ "type": "st1", }, - claimSize: "500Gi", // minimum for st1 - expectedSize: "500Gi", - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "500Gi", // minimum for st1 + ExpectedSize: "500Gi", + PvCheck: func(volume *v1.PersistentVolume) error { return checkAWSEBS(volume, "st1", false) }, }, { - name: "encrypted EBS on AWS", - cloudProviders: []string{"aws"}, - provisioner: "kubernetes.io/aws-ebs", - parameters: map[string]string{ + Name: "encrypted EBS on AWS", + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + Parameters: map[string]string{ "encrypted": "true", }, - claimSize: "1Gi", - expectedSize: "1Gi", - pvCheck: func(volume *v1.PersistentVolume) error { + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + PvCheck: func(volume *v1.PersistentVolume) error { return checkAWSEBS(volume, "gp2", true) }, }, // OpenStack generic tests (works on all OpenStack deployments) { - name: "generic Cinder volume on OpenStack", - cloudProviders: []string{"openstack"}, - provisioner: "kubernetes.io/cinder", - parameters: map[string]string{}, - claimSize: "1.5Gi", - expectedSize: "2Gi", - pvCheck: nil, // there is currently nothing to check on OpenStack + Name: "generic Cinder volume on OpenStack", + CloudProviders: []string{"openstack"}, + Provisioner: "kubernetes.io/cinder", + Parameters: map[string]string{}, + ClaimSize: "1.5Gi", + ExpectedSize: "2Gi", + PvCheck: nil, // there is currently nothing to check on OpenStack }, { - name: "Cinder volume with empty volume type and zone on OpenStack", - cloudProviders: []string{"openstack"}, - provisioner: "kubernetes.io/cinder", - parameters: map[string]string{ + Name: "Cinder volume with empty volume type and zone on OpenStack", + CloudProviders: []string{"openstack"}, + Provisioner: "kubernetes.io/cinder", + Parameters: map[string]string{ "type": "", "availability": "", }, - claimSize: "1.5Gi", - expectedSize: "2Gi", - pvCheck: nil, // there is currently nothing to check on OpenStack + ClaimSize: "1.5Gi", + ExpectedSize: "2Gi", + PvCheck: nil, // there is currently nothing to check on OpenStack }, // vSphere generic test { - name: "generic vSphere volume", - cloudProviders: []string{"vsphere"}, - provisioner: "kubernetes.io/vsphere-volume", - parameters: map[string]string{}, - claimSize: "1.5Gi", - expectedSize: "1.5Gi", - pvCheck: nil, + Name: "generic vSphere volume", + CloudProviders: []string{"vsphere"}, + Provisioner: "kubernetes.io/vsphere-volume", + Parameters: map[string]string{}, + ClaimSize: "1.5Gi", + ExpectedSize: "1.5Gi", + PvCheck: nil, }, // Azure { - name: "Azure disk volume with empty sku and location", - cloudProviders: []string{"azure"}, - provisioner: "kubernetes.io/azure-disk", - parameters: map[string]string{}, - claimSize: "1Gi", - expectedSize: "1Gi", - pvCheck: nil, + Name: "Azure disk volume with empty sku and location", + CloudProviders: []string{"azure"}, + Provisioner: "kubernetes.io/azure-disk", + Parameters: map[string]string{}, + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + PvCheck: nil, }, } - var betaTest *storageClassTest + var betaTest *testsuites.StorageClassTest for i, t := range tests { // Beware of clojure, use local variables instead of those from // outer scope test := t - if !framework.ProviderIs(test.cloudProviders...) { - framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders) + if !framework.ProviderIs(test.CloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) continue } // Remember the last supported test for subsequent test of beta API betaTest = &test - By("Testing " + test.name) + By("Testing " + test.Name) suffix := fmt.Sprintf("%d", i) class := newStorageClass(test, ns, suffix) claim := newClaim(test, ns, suffix) claim.Spec.StorageClassName = &class.Name - testDynamicProvisioning(test, c, claim, class) + testsuites.TestDynamicProvisioning(test, c, claim, class) } // Run the last test with storage.k8s.io/v1beta1 on pvc if betaTest != nil { - By("Testing " + betaTest.name + " with beta volume provisioning") + By("Testing " + betaTest.Name + " with beta volume provisioning") class := newBetaStorageClass(*betaTest, "beta") // we need to create the class manually, testDynamicProvisioning does not accept beta class class, err := c.StorageV1beta1().StorageClasses().Create(class) @@ -541,67 +423,10 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { claim := newClaim(*betaTest, ns, "beta") claim.Spec.StorageClassName = &(class.Name) - testDynamicProvisioning(*betaTest, c, claim, nil) + testsuites.TestDynamicProvisioning(*betaTest, c, claim, nil) } }) - It("should provision storage with non-default reclaim policy Retain", func() { - framework.SkipUnlessProviderIs("gce", "gke") - - test := storageClassTest{ - name: "HDD PD on GCE/GKE", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ - "type": "pd-standard", - }, - claimSize: "1Gi", - expectedSize: "1Gi", - pvCheck: func(volume *v1.PersistentVolume) error { - return checkGCEPD(volume, "pd-standard") - }, - } - class := newStorageClass(test, ns, "reclaimpolicy") - retain := v1.PersistentVolumeReclaimRetain - class.ReclaimPolicy = &retain - claim := newClaim(test, ns, "reclaimpolicy") - claim.Spec.StorageClassName = &class.Name - pv := testDynamicProvisioning(test, c, claim, class) - - By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased)) - framework.ExpectNoError(framework.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second)) - - By(fmt.Sprintf("deleting the storage asset backing the PV %q", pv.Name)) - framework.ExpectNoError(framework.DeletePDWithRetry(pv.Spec.GCEPersistentDisk.PDName)) - - By(fmt.Sprintf("deleting the PV %q", pv.Name)) - framework.ExpectNoError(framework.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name) - framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(c, pv.Name, 1*time.Second, 30*time.Second)) - }) - - It("should provision storage with mount options", func() { - framework.SkipUnlessProviderIs("gce", "gke") - - test := storageClassTest{ - name: "HDD PD on GCE/GKE", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{ - "type": "pd-standard", - }, - claimSize: "1Gi", - expectedSize: "1Gi", - pvCheck: func(volume *v1.PersistentVolume) error { - return checkGCEPD(volume, "pd-standard") - }, - } - class := newStorageClass(test, ns, "mountoptions") - class.MountOptions = []string{"debug", "nouid32"} - claim := newClaim(test, ns, "mountoptions") - claim.Spec.StorageClassName = &class.Name - testDynamicProvisioning(test, c, claim, class) - }) - It("should not provision a volume in an unmanaged GCE zone.", func() { framework.SkipUnlessProviderIs("gce", "gke") var suffix string = "unmananged" @@ -634,11 +459,11 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { } By("Creating a StorageClass for the unmanaged zone") - test := storageClassTest{ - name: "unmanaged_zone", - provisioner: "kubernetes.io/gce-pd", - parameters: map[string]string{"zone": unmanagedZone}, - claimSize: "1Gi", + test := testsuites.StorageClassTest{ + Name: "unmanaged_zone", + Provisioner: "kubernetes.io/gce-pd", + Parameters: map[string]string{"zone": unmanagedZone}, + ClaimSize: "1Gi", } sc := newStorageClass(test, ns, suffix) sc, err = c.StorageV1().StorageClasses().Create(sc) @@ -671,10 +496,10 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { const raceAttempts int = 100 var residualPVs []*v1.PersistentVolume By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts)) - test := storageClassTest{ - name: "deletion race", - provisioner: "", // Use a native one based on current cloud provider - claimSize: "1Gi", + test := testsuites.StorageClassTest{ + Name: "deletion race", + Provisioner: "", // Use a native one based on current cloud provider + ClaimSize: "1Gi", } class := newStorageClass(test, ns, "race") @@ -822,18 +647,18 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { defer framework.DeletePodOrFail(c, ns, pod.Name) By("creating a StorageClass") - test := storageClassTest{ - name: "external provisioner test", - provisioner: externalPluginName, - claimSize: "1500Mi", - expectedSize: "1500Mi", + test := testsuites.StorageClassTest{ + Name: "external provisioner test", + Provisioner: externalPluginName, + ClaimSize: "1500Mi", + ExpectedSize: "1500Mi", } class := newStorageClass(test, ns, "external") claim := newClaim(test, ns, "external") claim.Spec.StorageClassName = &(class.Name) By("creating a claim with a external provisioning annotation") - testDynamicProvisioning(test, c, claim, class) + testsuites.TestDynamicProvisioning(test, c, claim, class) }) }) @@ -842,23 +667,23 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure") By("creating a claim with no annotation") - test := storageClassTest{ - name: "default", - claimSize: "2Gi", - expectedSize: "2Gi", + test := testsuites.StorageClassTest{ + Name: "default", + ClaimSize: "2Gi", + ExpectedSize: "2Gi", } claim := newClaim(test, ns, "default") - testDynamicProvisioning(test, c, claim, nil) + testsuites.TestDynamicProvisioning(test, c, claim, nil) }) // Modifying the default storage class can be disruptive to other tests that depend on it It("should be disabled by changing the default annotation [Serial] [Disruptive]", func() { framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure") scName := getDefaultStorageClassName(c) - test := storageClassTest{ - name: "default", - claimSize: "2Gi", + test := testsuites.StorageClassTest{ + Name: "default", + ClaimSize: "2Gi", } By("setting the is-default StorageClass annotation to false") @@ -887,9 +712,9 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { It("should be disabled by removing the default annotation [Serial] [Disruptive]", func() { framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure") scName := getDefaultStorageClassName(c) - test := storageClassTest{ - name: "default", - claimSize: "2Gi", + test := testsuites.StorageClassTest{ + Name: "default", + ClaimSize: "2Gi", } By("removing the is-default StorageClass annotation") @@ -921,13 +746,13 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { pod := startGlusterDpServerPod(c, ns) serverUrl := "https://" + pod.Status.PodIP + ":8081" By("creating a StorageClass") - test := storageClassTest{ - name: "Gluster Dynamic provisioner test", - provisioner: "kubernetes.io/glusterfs", - claimSize: "2Gi", - expectedSize: "2Gi", - parameters: map[string]string{"resturl": serverUrl}, - skipWriteReadCheck: true, + test := testsuites.StorageClassTest{ + Name: "Gluster Dynamic provisioner test", + Provisioner: "kubernetes.io/glusterfs", + ClaimSize: "2Gi", + ExpectedSize: "2Gi", + Parameters: map[string]string{"resturl": serverUrl}, + SkipWriteReadCheck: true, } suffix := fmt.Sprintf("glusterdptest") class := newStorageClass(test, ns, suffix) @@ -935,38 +760,18 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { By("creating a claim object with a suffix for gluster dynamic provisioner") claim := newClaim(test, ns, suffix) - testDynamicProvisioning(test, c, claim, class) + testsuites.TestDynamicProvisioning(test, c, claim, class) }) }) - Describe("Block volume provisioning [Feature:BlockVolume]", func() { - It("should create and delete block persistent volumes", func() { - - // TODO: add openstack once Cinder volume plugin supports block volumes - framework.SkipUnlessProviderIs("gce", "aws", "gke", "vsphere", "azure") - - By("creating a claim with default class") - block := v1.PersistentVolumeBlock - test := storageClassTest{ - name: "default", - claimSize: "2Gi", - expectedSize: "2Gi", - volumeMode: &block, - skipWriteReadCheck: true, - } - claim := newClaim(test, ns, "default") - claim.Spec.VolumeMode = &block - testDynamicProvisioning(test, c, claim, nil) - }) - }) Describe("Invalid AWS KMS key", func() { It("should report an error and create no PV", func() { framework.SkipUnlessProviderIs("aws") - test := storageClassTest{ - name: "AWS EBS with invalid KMS key", - provisioner: "kubernetes.io/aws-ebs", - claimSize: "2Gi", - parameters: map[string]string{"kmsKeyId": "arn:aws:kms:us-east-1:123456789012:key/55555555-5555-5555-5555-555555555555"}, + test := testsuites.StorageClassTest{ + Name: "AWS EBS with invalid KMS key", + Provisioner: "kubernetes.io/aws-ebs", + ClaimSize: "2Gi", + Parameters: map[string]string{"kmsKeyId": "arn:aws:kms:us-east-1:123456789012:key/55555555-5555-5555-5555-555555555555"}, } By("creating a StorageClass") @@ -1008,25 +813,25 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { }) Describe("DynamicProvisioner delayed binding [Slow]", func() { It("should create a persistent volume in the same zone as node after a pod mounting the claim is started", func() { - tests := []storageClassTest{ + tests := []testsuites.StorageClassTest{ { - name: "Delayed binding EBS storage class test", - cloudProviders: []string{"aws"}, - provisioner: "kubernetes.io/aws-ebs", - claimSize: "2Gi", - delayBinding: true, + Name: "Delayed binding EBS storage class test", + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + ClaimSize: "2Gi", + DelayBinding: true, }, { - name: "Delayed binding GCE PD storage class test", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - claimSize: "2Gi", - delayBinding: true, + Name: "Delayed binding GCE PD storage class test", + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + ClaimSize: "2Gi", + DelayBinding: true, }, } for _, test := range tests { - if !framework.ProviderIs(test.cloudProviders...) { - framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders) + if !framework.ProviderIs(test.CloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) continue } By("creating a claim with class with waitForFirstConsumer") @@ -1048,25 +853,25 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { }) Describe("DynamicProvisioner allowedTopologies", func() { It("should create persistent volume in the zone specified in allowedTopologies of storageclass", func() { - tests := []storageClassTest{ + tests := []testsuites.StorageClassTest{ { - name: "AllowedTopologies EBS storage class test", - cloudProviders: []string{"aws"}, - provisioner: "kubernetes.io/aws-ebs", - claimSize: "2Gi", - expectedSize: "2Gi", + Name: "AllowedTopologies EBS storage class test", + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + ClaimSize: "2Gi", + ExpectedSize: "2Gi", }, { - name: "AllowedTopologies GCE PD storage class test", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - claimSize: "2Gi", - expectedSize: "2Gi", + Name: "AllowedTopologies GCE PD storage class test", + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + ClaimSize: "2Gi", + ExpectedSize: "2Gi", }, } for _, test := range tests { - if !framework.ProviderIs(test.cloudProviders...) { - framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders) + if !framework.ProviderIs(test.CloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) continue } By("creating a claim with class with allowedTopologies set") @@ -1076,32 +881,32 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { addSingleZoneAllowedTopologyToStorageClass(c, class, zone) claim := newClaim(test, ns, suffix) claim.Spec.StorageClassName = &class.Name - pv := testDynamicProvisioning(test, c, claim, class) + pv := testsuites.TestDynamicProvisioning(test, c, claim, class) checkZoneFromLabelAndAffinity(pv, zone, true) } }) }) Describe("DynamicProvisioner delayed binding with allowedTopologies [Slow]", func() { It("should create persistent volume in the same zone as specified in allowedTopologies after a pod mounting the claim is started", func() { - tests := []storageClassTest{ + tests := []testsuites.StorageClassTest{ { - name: "AllowedTopologies and delayed binding EBS storage class test", - cloudProviders: []string{"aws"}, - provisioner: "kubernetes.io/aws-ebs", - claimSize: "2Gi", - delayBinding: true, + Name: "AllowedTopologies and delayed binding EBS storage class test", + CloudProviders: []string{"aws"}, + Provisioner: "kubernetes.io/aws-ebs", + ClaimSize: "2Gi", + DelayBinding: true, }, { - name: "AllowedTopologies and delayed binding GCE PD storage class test", - cloudProviders: []string{"gce", "gke"}, - provisioner: "kubernetes.io/gce-pd", - claimSize: "2Gi", - delayBinding: true, + Name: "AllowedTopologies and delayed binding GCE PD storage class test", + CloudProviders: []string{"gce", "gke"}, + Provisioner: "kubernetes.io/gce-pd", + ClaimSize: "2Gi", + DelayBinding: true, }, } for _, test := range tests { - if !framework.ProviderIs(test.cloudProviders...) { - framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders) + if !framework.ProviderIs(test.CloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders) continue } By("creating a claim with class with WaitForFirstConsumer and allowedTopologies") @@ -1202,59 +1007,8 @@ func getClaim(claimSize string, ns string) *v1.PersistentVolumeClaim { return &claim } -func newClaim(t storageClassTest, ns, suffix string) *v1.PersistentVolumeClaim { - return getClaim(t.claimSize, ns) -} - -// runInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory. -func runInPodWithVolume(c clientset.Interface, ns, claimName, nodeName, command string) { - pod := &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pvc-volume-tester-", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "volume-tester", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"/bin/sh"}, - Args: []string{"-c", command}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "my-volume", - MountPath: "/mnt/test", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "my-volume", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: claimName, - ReadOnly: false, - }, - }, - }, - }, - }, - } - - if len(nodeName) != 0 { - pod.Spec.NodeName = nodeName - } - pod, err := c.CoreV1().Pods(ns).Create(pod) - framework.ExpectNoError(err, "Failed to create pod: %v", err) - defer func() { - framework.DeletePodOrFail(c, ns, pod.Name) - }() - framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace)) +func newClaim(t testsuites.StorageClassTest, ns, suffix string) *v1.PersistentVolumeClaim { + return getClaim(t.ClaimSize, ns) } func getDefaultPluginName() string { @@ -1285,8 +1039,8 @@ func addSingleZoneAllowedTopologyToStorageClass(c clientset.Interface, sc *stora sc.AllowedTopologies = append(sc.AllowedTopologies, term) } -func newStorageClass(t storageClassTest, ns string, suffix string) *storage.StorageClass { - pluginName := t.provisioner +func newStorageClass(t testsuites.StorageClassTest, ns string, suffix string) *storage.StorageClass { + pluginName := t.Provisioner if pluginName == "" { pluginName = getDefaultPluginName() } @@ -1294,10 +1048,10 @@ func newStorageClass(t storageClassTest, ns string, suffix string) *storage.Stor suffix = "sc" } bindingMode := storage.VolumeBindingImmediate - if t.delayBinding { + if t.DelayBinding { bindingMode = storage.VolumeBindingWaitForFirstConsumer } - return getStorageClass(pluginName, t.parameters, &bindingMode, ns, suffix) + return getStorageClass(pluginName, t.Parameters, &bindingMode, ns, suffix) } func getStorageClass( @@ -1326,8 +1080,8 @@ func getStorageClass( } // TODO: remove when storage.k8s.io/v1beta1 is removed. -func newBetaStorageClass(t storageClassTest, suffix string) *storagebeta.StorageClass { - pluginName := t.provisioner +func newBetaStorageClass(t testsuites.StorageClassTest, suffix string) *storagebeta.StorageClass { + pluginName := t.Provisioner if pluginName == "" { pluginName = getDefaultPluginName() @@ -1344,7 +1098,7 @@ func newBetaStorageClass(t storageClassTest, suffix string) *storagebeta.Storage GenerateName: suffix + "-", }, Provisioner: pluginName, - Parameters: t.parameters, + Parameters: t.Parameters, } }