diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index 5e7c6d9a6f6..b07dc69dfa0 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ - "csi_objects.go", "csi_volumes.go", "empty_dir_wrapper.go", "ephemeral_volume.go", @@ -72,6 +71,7 @@ go_library( "//test/e2e/framework/providers/gce:go_default_library", "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/storage/drivers:go_default_library", + "//test/e2e/storage/testpatterns:go_default_library", "//test/e2e/storage/testsuites:go_default_library", "//test/e2e/storage/utils:go_default_library", "//test/utils/image:go_default_library", diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 24ac6658219..7adcfc655c3 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -19,9 +19,7 @@ package storage import ( "context" "fmt" - "math/rand" "regexp" - "time" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -32,6 +30,8 @@ import ( csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/podlogs" + "k8s.io/kubernetes/test/e2e/storage/drivers" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -42,39 +42,56 @@ import ( . "github.com/onsi/gomega" ) -type csiTestDriver interface { - createCSIDriver() - cleanupCSIDriver() - createStorageClassTest() testsuites.StorageClassTest +// List of testDrivers to be executed in below loop +var csiTestDrivers = []func() drivers.TestDriver{ + drivers.InitHostPathCSIDriver, + drivers.InitGcePDCSIDriver, + drivers.InitGcePDExternalCSIDriver, } -var csiTestDrivers = map[string]func(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver{ - "hostPath": initCSIHostpath, - "gcePD": initCSIgcePD, - // TODO(#70258): this is temporary until we can figure out how to make e2e tests a library - "[Feature: gcePD-external]": initCSIgcePDExternal, +// List of testSuites to be executed in below loop +var csiTestSuites = []func() testsuites.TestSuite{ + testsuites.InitVolumesTestSuite, + testsuites.InitVolumeIOTestSuite, + testsuites.InitVolumeModeTestSuite, + testsuites.InitSubPathTestSuite, + testsuites.InitProvisioningTestSuite, } +func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern { + tunedPatterns := []testpatterns.TestPattern{} + + for _, pattern := range patterns { + // Skip inline volume and pre-provsioned PV tests for csi drivers + if pattern.VolType == testpatterns.InlineVolume || pattern.VolType == testpatterns.PreprovisionedPV { + continue + } + tunedPatterns = append(tunedPatterns, pattern) + } + + return tunedPatterns +} + +// This executes testSuites for csi volumes. var _ = utils.SIGDescribe("CSI Volumes", func() { f := framework.NewDefaultFramework("csi-volumes") var ( cancel context.CancelFunc cs clientset.Interface - csics csiclient.Interface ns *v1.Namespace - node v1.Node config framework.VolumeTestConfig ) BeforeEach(func() { ctx, c := context.WithCancel(context.Background()) cancel = c - cs = f.ClientSet - csics = f.CSIClientSet ns = f.Namespace - + config = framework.VolumeTestConfig{ + Namespace: ns.Name, + Prefix: "csi", + } // Debugging of the following tests heavily depends on the log output // of the different containers. Therefore include all of that in log // files (when using --report-dir, as in the CI) or the output stream @@ -100,69 +117,50 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { if framework.TestContext.ReportDir == "" { podlogs.WatchPods(ctx, cs, ns.Name, GinkgoWriter) } - - nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) - node = nodes.Items[rand.Intn(len(nodes.Items))] - config = framework.VolumeTestConfig{ - Namespace: ns.Name, - Prefix: "csi", - // TODO(#70259): this needs to be parameterized so only hostpath sets node name - ClientNodeName: node.Name, - ServerNodeName: node.Name, - WaitForCompletion: true, - } }) AfterEach(func() { cancel() }) - for driverName, initCSIDriver := range csiTestDrivers { - curDriverName := driverName - curInitCSIDriver := initCSIDriver - - Context(fmt.Sprintf("CSI plugin test using CSI driver: %s", curDriverName), func() { - var ( - driver csiTestDriver - ) + for _, initDriver := range csiTestDrivers { + curDriver := initDriver() + Context(fmt.Sprintf(drivers.GetDriverNameWithFeatureTags(curDriver)), func() { + driver := curDriver BeforeEach(func() { - driver = curInitCSIDriver(f, config) - driver.createCSIDriver() + // setupDriver + drivers.SetCommonDriverParameters(driver, f, config) + driver.CreateDriver() }) AfterEach(func() { - driver.cleanupCSIDriver() + // Cleanup driver + driver.CleanupDriver() }) - It("should provision storage", func() { - t := driver.createStorageClassTest() - claim := newClaim(t, ns.GetName(), "") - var class *storagev1.StorageClass - if t.StorageClassName == "" { - class = newStorageClass(t, ns.GetName(), "") - claim.Spec.StorageClassName = &class.ObjectMeta.Name - } else { - scName := t.StorageClassName - claim.Spec.StorageClassName = &scName - } - testsuites.TestDynamicProvisioning(t, cs, claim, class) - }) + testsuites.RunTestSuite(f, config, driver, csiTestSuites, csiTunePattern) }) } // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. Context("CSI attach test using HostPath driver [Feature:CSISkipAttach]", func() { var ( - driver csiTestDriver + cs clientset.Interface + csics csiclient.Interface + driver drivers.TestDriver ) + BeforeEach(func() { - driver = initCSIHostpath(f, config) - driver.createCSIDriver() + cs = f.ClientSet + csics = f.CSIClientSet + driver = drivers.InitHostPathCSIDriver() + drivers.SetCommonDriverParameters(driver, f, config) + driver.CreateDriver() }) AfterEach(func() { - driver.cleanupCSIDriver() + driver.CleanupDriver() }) tests := []struct { @@ -194,15 +192,27 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { test := t It(test.name, func() { if test.driverExists { - driver := createCSIDriver(csics, "csi-hostpath-"+f.UniqueName, test.driverAttachable) - if driver != nil { - defer csics.CsiV1alpha1().CSIDrivers().Delete(driver.Name, nil) + csiDriver := createCSIDriver(csics, "csi-hostpath-"+f.UniqueName, test.driverAttachable) + if csiDriver != nil { + defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil) } } By("Creating pod") - t := driver.createStorageClassTest() - class, claim, pod := startPausePod(cs, t, ns.Name) + var sc *storagev1.StorageClass + if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { + sc = dDriver.GetDynamicProvisionStorageClass("") + } + nodeName := driver.GetDriverInfo().Config.ClientNodeName + scTest := testsuites.StorageClassTest{ + Name: driver.GetDriverInfo().Name, + Provisioner: sc.Provisioner, + Parameters: sc.Parameters, + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + NodeName: nodeName, + } + class, claim, pod := startPausePod(cs, scTest, ns.Name) if class != nil { defer cs.StorageV1().StorageClasses().Delete(class.Name, nil) } @@ -223,7 +233,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { By("Checking if VolumeAttachment was created for the pod") // Check that VolumeAttachment does not exist handle := getVolumeHandle(cs, claim) - attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, t.Provisioner, node.Name))) + attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, scTest.Provisioner, nodeName))) attachmentName := fmt.Sprintf("csi-%x", attachmentHash) _, err = cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{}) if err != nil { @@ -326,164 +336,3 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, ns str framework.ExpectNoError(err, "Failed to create pod: %v", err) return class, claim, pod } - -type hostpathCSIDriver struct { - f *framework.Framework - config framework.VolumeTestConfig - cleanup func() -} - -func initCSIHostpath(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver { - return &hostpathCSIDriver{ - f: f, - config: config, - } -} - -func (h *hostpathCSIDriver) createStorageClassTest() testsuites.StorageClassTest { - return testsuites.StorageClassTest{ - Name: "csi-hostpath", - Parameters: map[string]string{}, - ClaimSize: "1Gi", - ExpectedSize: "1Gi", - - // The hostpath driver only works when everything runs on a single node. - NodeName: h.config.ServerNodeName, - - // Provisioner and storage class name must match what's used in - // csi-storageclass.yaml, plus the test-specific suffix. - Provisioner: "csi-hostpath-" + h.f.UniqueName, - StorageClassName: "csi-hostpath-sc-" + h.f.UniqueName, - } -} - -func (h *hostpathCSIDriver) createCSIDriver() { - By("deploying csi hostpath driver") - // TODO (?): the storage.csi.image.version and storage.csi.image.registry - // settings are ignored for this test. We could patch the image definitions. - o := utils.PatchCSIOptions{ - OldDriverName: "csi-hostpath", - NewDriverName: "csi-hostpath-" + h.f.UniqueName, - DriverContainerName: "hostpath", - ProvisionerContainerName: "csi-provisioner", - NodeName: h.config.ServerNodeName, - } - cleanup, err := h.f.CreateFromManifests(func(item interface{}) error { - return utils.PatchCSIDeployment(h.f, o, item) - }, - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml", - ) - h.cleanup = cleanup - if err != nil { - framework.Failf("deploying csi hostpath driver: %v", err) - } -} - -func (h *hostpathCSIDriver) cleanupCSIDriver() { - if h.cleanup != nil { - By("uninstalling csi hostpath driver") - h.cleanup() - } -} - -type gcePDCSIDriver struct { - f *framework.Framework - config framework.VolumeTestConfig - cleanup func() -} - -func initCSIgcePD(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver { - cs := f.ClientSet - framework.SkipUnlessProviderIs("gce", "gke") - framework.SkipIfMultizone(cs) - - // TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys. - createGCESecrets(cs, config) - - framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute) - - return &gcePDCSIDriver{ - f: f, - config: config, - } -} - -func (g *gcePDCSIDriver) createStorageClassTest() testsuites.StorageClassTest { - return testsuites.StorageClassTest{ - Name: "com.google.csi.gcepd", - // *Not* renaming the driver, see below. - Provisioner: "com.google.csi.gcepd", - Parameters: map[string]string{"type": "pd-standard"}, - ClaimSize: "5Gi", - ExpectedSize: "5Gi", - } -} - -func (g *gcePDCSIDriver) createCSIDriver() { - By("deploying gce-pd driver") - // It would be safer to rename the gcePD driver, but that - // hasn't been done before either and attempts to do so now led to - // errors during driver registration, therefore it is disabled - // by passing a nil function below. - // - // These are the options which would have to be used: - // o := utils.PatchCSIOptions{ - // OldDriverName: "com.google.csi.gcepd", - // NewDriverName: "com.google.csi.gcepd-" + g.f.UniqueName, - // DriverContainerName: "gce-driver", - // ProvisionerContainerName: "csi-external-provisioner", - // } - cleanup, err := g.f.CreateFromManifests(nil, - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml", - "test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml", - "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml", - ) - g.cleanup = cleanup - if err != nil { - framework.Failf("deploying csi hostpath driver: %v", err) - } -} - -func (g *gcePDCSIDriver) cleanupCSIDriver() { - By("uninstalling gce-pd driver") - if g.cleanup != nil { - g.cleanup() - } -} - -type gcePDCSIDriverExternal struct { -} - -func initCSIgcePDExternal(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver { - cs := f.ClientSet - framework.SkipUnlessProviderIs("gce", "gke") - framework.SkipIfMultizone(cs) - - return &gcePDCSIDriverExternal{} -} - -func (g *gcePDCSIDriverExternal) createStorageClassTest() testsuites.StorageClassTest { - return testsuites.StorageClassTest{ - Name: "com.google.csi.gcepd", - Provisioner: "com.google.csi.gcepd", - Parameters: map[string]string{"type": "pd-standard"}, - ClaimSize: "5Gi", - ExpectedSize: "5Gi", - } -} - -func (g *gcePDCSIDriverExternal) createCSIDriver() { -} - -func (g *gcePDCSIDriverExternal) cleanupCSIDriver() { -} diff --git a/test/e2e/storage/drivers/BUILD b/test/e2e/storage/drivers/BUILD index 72beb967aae..487e0f21866 100644 --- a/test/e2e/storage/drivers/BUILD +++ b/test/e2e/storage/drivers/BUILD @@ -4,6 +4,8 @@ go_library( name = "go_default_library", srcs = [ "base.go", + "csi.go", + "csi_objects.go", "in_tree.go", ], importpath = "k8s.io/kubernetes/test/e2e/storage/drivers", @@ -17,7 +19,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/storage/testpatterns:go_default_library", "//test/e2e/storage/utils:go_default_library", diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go new file mode 100644 index 00000000000..d32dee75a2c --- /dev/null +++ b/test/e2e/storage/drivers/csi.go @@ -0,0 +1,281 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + * This file defines various csi volume test drivers for TestSuites. + * + * There are two ways, how to prepare test drivers: + * 1) With containerized server (NFS, Ceph, Gluster, iSCSI, ...) + * It creates a server pod which defines one volume for the tests. + * These tests work only when privileged containers are allowed, exporting + * various filesystems (NFS, GlusterFS, ...) usually needs some mounting or + * other privileged magic in the server pod. + * + * Note that the server containers are for testing purposes only and should not + * be used in production. + * + * 2) With server or cloud provider outside of Kubernetes (Cinder, GCE, AWS, Azure, ...) + * Appropriate server or cloud provider must exist somewhere outside + * the tested Kubernetes cluster. CreateVolume will create a new volume to be + * used in the TestSuites for inlineVolume or DynamicPV tests. + */ + +package drivers + +import ( + "fmt" + "math/rand" + "time" + + . "github.com/onsi/ginkgo" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" + "k8s.io/kubernetes/test/e2e/storage/utils" +) + +// hostpathCSI +type hostpathCSIDriver struct { + cleanup func() + driverInfo DriverInfo +} + +var _ TestDriver = &hostpathCSIDriver{} +var _ DynamicPVTestDriver = &hostpathCSIDriver{} + +// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface +func InitHostPathCSIDriver() TestDriver { + return &hostpathCSIDriver{ + driverInfo: DriverInfo{ + Name: "csi-hostpath", + FeatureTag: "", + MaxFileSize: testpatterns.FileSizeMedium, + SupportedFsType: sets.NewString( + "", // Default fsType + ), + IsPersistent: true, + IsFsGroupSupported: false, + IsBlockSupported: false, + }, + } +} + +func (h *hostpathCSIDriver) GetDriverInfo() *DriverInfo { + return &h.driverInfo +} + +func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { +} + +func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { + provisioner := h.driverInfo.Name + h.driverInfo.Framework.UniqueName + parameters := map[string]string{} + ns := h.driverInfo.Framework.Namespace.Name + suffix := fmt.Sprintf("%s-sc", provisioner) + + return getStorageClass(provisioner, parameters, nil, ns, suffix) +} + +func (h *hostpathCSIDriver) CreateDriver() { + By("deploying csi hostpath driver") + f := h.driverInfo.Framework + cs := f.ClientSet + + // pods should be scheduled on the node + nodes := framework.GetReadySchedulableNodesOrDie(cs) + node := nodes.Items[rand.Intn(len(nodes.Items))] + h.driverInfo.Config.ClientNodeName = node.Name + h.driverInfo.Config.ServerNodeName = node.Name + + // TODO (?): the storage.csi.image.version and storage.csi.image.registry + // settings are ignored for this test. We could patch the image definitions. + o := utils.PatchCSIOptions{ + OldDriverName: h.driverInfo.Name, + NewDriverName: h.driverInfo.Name + h.driverInfo.Framework.UniqueName, + DriverContainerName: "hostpath", + ProvisionerContainerName: "csi-provisioner", + NodeName: h.driverInfo.Config.ServerNodeName, + } + cleanup, err := h.driverInfo.Framework.CreateFromManifests(func(item interface{}) error { + return utils.PatchCSIDeployment(h.driverInfo.Framework, o, item) + }, + "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", + ) + h.cleanup = cleanup + if err != nil { + framework.Failf("deploying csi hostpath driver: %v", err) + } +} + +func (h *hostpathCSIDriver) CleanupDriver() { + if h.cleanup != nil { + By("uninstalling csi hostpath driver") + h.cleanup() + } +} + +// gce-pd +type gcePDCSIDriver struct { + cleanup func() + driverInfo DriverInfo +} + +var _ TestDriver = &gcePDCSIDriver{} +var _ DynamicPVTestDriver = &gcePDCSIDriver{} + +// InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface +func InitGcePDCSIDriver() TestDriver { + return &gcePDCSIDriver{ + driverInfo: DriverInfo{ + Name: "com.google.csi.gcepd", + FeatureTag: "[Serial]", + MaxFileSize: testpatterns.FileSizeMedium, + SupportedFsType: sets.NewString( + "", // Default fsType + "ext2", + "ext3", + "ext4", + "xfs", + ), + IsPersistent: true, + IsFsGroupSupported: true, + IsBlockSupported: false, + }, + } +} + +func (g *gcePDCSIDriver) GetDriverInfo() *DriverInfo { + return &g.driverInfo +} + +func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { + f := g.driverInfo.Framework + cs := f.ClientSet + config := g.driverInfo.Config + framework.SkipUnlessProviderIs("gce", "gke") + framework.SkipIfMultizone(cs) + + // TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys. + createGCESecrets(cs, config) + framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute) +} + +func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { + ns := g.driverInfo.Framework.Namespace.Name + provisioner := g.driverInfo.Name + suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) + + parameters := map[string]string{"type": "pd-standard"} + + return getStorageClass(provisioner, parameters, nil, ns, suffix) +} + +func (g *gcePDCSIDriver) CreateDriver() { + By("deploying csi gce-pd driver") + // It would be safer to rename the gcePD driver, but that + // hasn't been done before either and attempts to do so now led to + // errors during driver registration, therefore it is disabled + // by passing a nil function below. + // + // These are the options which would have to be used: + // o := utils.PatchCSIOptions{ + // OldDriverName: "com.google.csi.gcepd", + // NewDriverName: "com.google.csi.gcepd-" + g.f.UniqueName, + // DriverContainerName: "gce-driver", + // ProvisionerContainerName: "csi-external-provisioner", + // } + cleanup, err := g.driverInfo.Framework.CreateFromManifests(nil, + "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml", + "test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml", + "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml", + ) + g.cleanup = cleanup + if err != nil { + framework.Failf("deploying csi gce-pd driver: %v", err) + } +} + +func (g *gcePDCSIDriver) CleanupDriver() { + By("uninstalling gce-pd driver") + if g.cleanup != nil { + g.cleanup() + } +} + +// gcePd-external +type gcePDExternalCSIDriver struct { + driverInfo DriverInfo +} + +var _ TestDriver = &gcePDExternalCSIDriver{} +var _ DynamicPVTestDriver = &gcePDExternalCSIDriver{} + +// InitGcePDExternalCSIDriver returns gcePDExternalCSIDriver that implements TestDriver interface +func InitGcePDExternalCSIDriver() TestDriver { + return &gcePDExternalCSIDriver{ + driverInfo: DriverInfo{ + Name: "com.google.csi.gcepd", + // TODO(#70258): this is temporary until we can figure out how to make e2e tests a library + FeatureTag: "[Serial][Feature: gcePD-external]", + MaxFileSize: testpatterns.FileSizeMedium, + SupportedFsType: sets.NewString( + "", // Default fsType + "ext2", + "ext3", + "ext4", + "xfs", + ), + IsPersistent: true, + IsFsGroupSupported: true, + IsBlockSupported: false, + }, + } +} + +func (g *gcePDExternalCSIDriver) GetDriverInfo() *DriverInfo { + return &g.driverInfo +} + +func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { + framework.SkipUnlessProviderIs("gce", "gke") + framework.SkipIfMultizone(g.driverInfo.Framework.ClientSet) +} + +func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { + ns := g.driverInfo.Framework.Namespace.Name + provisioner := g.driverInfo.Name + suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) + + parameters := map[string]string{"type": "pd-standard"} + + return getStorageClass(provisioner, parameters, nil, ns, suffix) +} + +func (g *gcePDExternalCSIDriver) CreateDriver() { +} + +func (g *gcePDExternalCSIDriver) CleanupDriver() { +} diff --git a/test/e2e/storage/csi_objects.go b/test/e2e/storage/drivers/csi_objects.go similarity index 99% rename from test/e2e/storage/csi_objects.go rename to test/e2e/storage/drivers/csi_objects.go index a7db54f0c81..cd3de2fce40 100644 --- a/test/e2e/storage/csi_objects.go +++ b/test/e2e/storage/drivers/csi_objects.go @@ -17,7 +17,7 @@ limitations under the License. // This file is used to deploy the CSI hostPath plugin // More Information: https://github.com/kubernetes-csi/drivers/tree/master/pkg/hostpath -package storage +package drivers import ( "flag" diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 2906a17cecc..6c6ea613686 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -23,6 +23,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/drivers" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -53,6 +54,10 @@ var testSuites = []func() testsuites.TestSuite{ testsuites.InitProvisioningTestSuite, } +func intreeTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern { + return patterns +} + // This executes testSuites for in-tree volumes. var _ = utils.SIGDescribe("In-tree Volumes", func() { f := framework.NewDefaultFramework("volumes") @@ -86,7 +91,7 @@ var _ = utils.SIGDescribe("In-tree Volumes", func() { driver.CleanupDriver() }) - testsuites.RunTestSuite(f, config, driver, testSuites) + testsuites.RunTestSuite(f, config, driver, testSuites, intreeTunePattern) }) } }) diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index fb8bde814f5..8d4f084989f 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -66,12 +66,12 @@ func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string { } // RunTestSuite runs all testpatterns of all testSuites for a driver -func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite) { +func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) { for _, testSuiteInit := range tsInits { suite := testSuiteInit() - tsInfo := suite.getTestSuiteInfo() + patterns := tunePatternFunc(suite.getTestSuiteInfo().testPatterns) - for _, pattern := range tsInfo.testPatterns { + for _, pattern := range patterns { suite.execTest(driver, pattern) } } @@ -164,7 +164,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat case testpatterns.DynamicPV: framework.Logf("Creating resource for dynamic PV") if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { - claimSize := "2Gi" + claimSize := "5Gi" r.sc = dDriver.GetDynamicProvisionStorageClass(fsType) By("creating a StorageClass " + r.sc.Name) diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 587b543512b..f897f54beb4 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -151,7 +151,7 @@ func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, patt framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name) } p.driver = driver - p.claimSize = "2Gi" + p.claimSize = "5Gi" p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name) p.pvc.Spec.StorageClassName = &p.sc.Name framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc) diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index 2b04f14d33d..619b1d545a0 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -19,6 +19,7 @@ package testsuites import ( "fmt" "path/filepath" + "regexp" "strings" "k8s.io/api/core/v1" @@ -315,7 +316,8 @@ func testSubPath(input *subPathTestInput) { }) It("should unmount if pod is force deleted while kubelet is down [Disruptive][Slow]", func() { - if input.volType == "hostPath" || input.volType == "hostPathSymlink" { + if strings.HasPrefix(input.volType, "hostPath") || strings.HasPrefix(input.volType, "csi-hostpath") { + // TODO: This skip should be removed once #61446 is fixed framework.Skipf("%s volume type does not support reconstruction, skipping", input.volType) } testSubpathReconstruction(input.f, input.pod, true) @@ -380,10 +382,23 @@ func TestBasicSubpathFile(f *framework.Framework, contents string, pod *v1.Pod, Expect(err).NotTo(HaveOccurred(), "while deleting pod") } +func generateSuffixForPodName(s string) string { + // Pod name must: + // 1. consist of lower case alphanumeric characters or '-', + // 2. start and end with an alphanumeric character. + // (e.g. 'my-name', or '123-abc', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?') + // Therefore, suffix is generated by following steps: + // 1. all strings other than [A-Za-z0-9] is replaced with "-", + // 2. add lower case alphanumeric characters at the end ('-[a-z0-9]{4}' is added), + // 3. convert the entire strings to lower case. + re := regexp.MustCompile("[^A-Za-z0-9]") + return strings.ToLower(fmt.Sprintf("%s-%s", re.ReplaceAllString(s, "-"), rand.String(4))) +} + // SubpathTestPod returns a pod spec for subpath tests func SubpathTestPod(f *framework.Framework, subpath, volumeType string, source *v1.VolumeSource, privilegedSecurityContext bool) *v1.Pod { var ( - suffix = strings.ToLower(fmt.Sprintf("%s-%s", volumeType, rand.String(4))) + suffix = generateSuffixForPodName(volumeType) gracePeriod = int64(1) probeVolumeName = "liveness-probe-volume" ) diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index cd9992fe153..360097b66f2 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -197,7 +197,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter } s.sc.VolumeBindingMode = &volBindMode - claimSize := "2Gi" + claimSize := "5Gi" s.pvc = getClaim(claimSize, ns.Name) s.pvc.Spec.StorageClassName = &s.sc.Name s.pvc.Spec.VolumeMode = &volMode