From 5349ce75fa42927afd1ff2ad5e1a903886f1b2c3 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 30 Nov 2018 15:10:23 +0100 Subject: [PATCH 1/5] e2e/storage: remove code duplication The setup of the V0 hostpath driver was done with copy-and-paste and then changing just the driver name and the manifests. The same can be achieved by making the base struct a bit more configurable, which simplifies future changes (less code). Renaming the provisioner container was unnecessary and was reverted to make it possible to use the same patch configuration. While at it, also fix the InitHostV0PathCSIDriver typo. --- test/e2e/storage/csi_volumes.go | 2 +- test/e2e/storage/drivers/csi.go | 120 ++++-------------- .../hostpath-v0/csi-hostpath-provisioner.yaml | 2 +- 3 files changed, 29 insertions(+), 95 deletions(-) diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index dd9a25806a7..e614f4e9513 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -47,7 +47,7 @@ var csiTestDrivers = []func() drivers.TestDriver{ drivers.InitHostPathCSIDriver, drivers.InitGcePDCSIDriver, drivers.InitGcePDExternalCSIDriver, - drivers.InitHostV0PathCSIDriver, + drivers.InitHostPathV0CSIDriver, } // List of testSuites to be executed in below loop diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index d58faa46f35..6bedcb71813 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -51,16 +51,13 @@ import ( type hostpathCSIDriver struct { cleanup func() driverInfo DriverInfo + manifests []string } -var _ TestDriver = &hostpathCSIDriver{} -var _ DynamicPVTestDriver = &hostpathCSIDriver{} - -// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface -func InitHostPathCSIDriver() TestDriver { +func initHostPathCSIDriver(name string, manifests ...string) TestDriver { return &hostpathCSIDriver{ driverInfo: DriverInfo{ - Name: "csi-hostpath", + Name: name, FeatureTag: "", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( @@ -70,9 +67,26 @@ func InitHostPathCSIDriver() TestDriver { CapPersistence: true, }, }, + manifests: manifests, } } +var _ TestDriver = &hostpathCSIDriver{} +var _ DynamicPVTestDriver = &hostpathCSIDriver{} + +// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface +func InitHostPathCSIDriver() TestDriver { + return initHostPathCSIDriver("csi-hostpath", + "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", + ) +} + func (h *hostpathCSIDriver) GetDriverInfo() *DriverInfo { return &h.driverInfo } @@ -90,7 +104,7 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *stor } func (h *hostpathCSIDriver) CreateDriver() { - By("deploying csi hostpath driver") + By(fmt.Sprintf("deploying %s driver", h.driverInfo.Name)) f := h.driverInfo.Framework cs := f.ClientSet @@ -112,92 +126,23 @@ func (h *hostpathCSIDriver) CreateDriver() { cleanup, err := h.driverInfo.Framework.CreateFromManifests(func(item interface{}) error { return utils.PatchCSIDeployment(h.driverInfo.Framework, o, item) }, - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", - ) + h.manifests...) h.cleanup = cleanup if err != nil { - framework.Failf("deploying csi hostpath driver: %v", err) + framework.Failf("deploying %s driver: %v", h.driverInfo.Name, err) } } func (h *hostpathCSIDriver) CleanupDriver() { if h.cleanup != nil { - By("uninstalling csi hostpath driver") + By(fmt.Sprintf("uninstalling %s driver", h.driverInfo.Name)) h.cleanup() } } -// hostpathV0CSIDriver -type hostpathV0CSIDriver struct { - cleanup func() - driverInfo DriverInfo -} - -var _ TestDriver = &hostpathV0CSIDriver{} -var _ DynamicPVTestDriver = &hostpathV0CSIDriver{} - -// InitHostPathV0CSIDriver returns hostpathV0CSIDriver that implements TestDriver interface -func InitHostV0PathCSIDriver() TestDriver { - return &hostpathV0CSIDriver{ - driverInfo: DriverInfo{ - Name: "csi-hostpath-v0", - FeatureTag: "", - MaxFileSize: testpatterns.FileSizeMedium, - SupportedFsType: sets.NewString( - "", // Default fsType - ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - }, - }, - } -} - -func (h *hostpathV0CSIDriver) GetDriverInfo() *DriverInfo { - return &h.driverInfo -} - -func (h *hostpathV0CSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { -} - -func (h *hostpathV0CSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { - provisioner := GetUniqueDriverName(h) - parameters := map[string]string{} - ns := h.driverInfo.Framework.Namespace.Name - suffix := fmt.Sprintf("%s-sc", provisioner) - - return getStorageClass(provisioner, parameters, nil, ns, suffix) -} - -func (h *hostpathV0CSIDriver) CreateDriver() { - By("deploying csi hostpath v0 driver") - f := h.driverInfo.Framework - cs := f.ClientSet - - // pods should be scheduled on the node - nodes := framework.GetReadySchedulableNodesOrDie(cs) - node := nodes.Items[rand.Intn(len(nodes.Items))] - h.driverInfo.Config.ClientNodeName = node.Name - h.driverInfo.Config.ServerNodeName = node.Name - - // TODO (?): the storage.csi.image.version and storage.csi.image.registry - // settings are ignored for this test. We could patch the image definitions. - o := utils.PatchCSIOptions{ - OldDriverName: h.driverInfo.Name, - NewDriverName: GetUniqueDriverName(h), - DriverContainerName: "hostpath", - ProvisionerContainerName: "csi-provisioner-v0", - NodeName: h.driverInfo.Config.ServerNodeName, - } - cleanup, err := h.driverInfo.Framework.CreateFromManifests(func(item interface{}) error { - return utils.PatchCSIDeployment(h.driverInfo.Framework, o, item) - }, +// InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests. +func InitHostPathV0CSIDriver() TestDriver { + return initHostPathCSIDriver("csi-hostpath-v0", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", @@ -206,17 +151,6 @@ func (h *hostpathV0CSIDriver) CreateDriver() { "test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpathplugin.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/e2e-test-rbac.yaml", ) - h.cleanup = cleanup - if err != nil { - framework.Failf("deploying csi hostpath v0 driver: %v", err) - } -} - -func (h *hostpathV0CSIDriver) CleanupDriver() { - if h.cleanup != nil { - By("uninstalling csi hostpath v0 driver") - h.cleanup() - } } // gce-pd diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpath-provisioner.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpath-provisioner.yaml index b9d4d2bfb9e..a7fefb5c516 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpath-provisioner.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpath-provisioner.yaml @@ -29,7 +29,7 @@ spec: spec: serviceAccountName: csi-provisioner containers: - - name: csi-provisioner-v0 + - name: csi-provisioner image: quay.io/k8scsi/csi-provisioner:v0.4.1 args: - "--provisioner=csi-hostpath-v0" From 41ab9449edaf6b434cf0a0d9dc60b0bdbad9dae9 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 9 Nov 2018 10:07:26 +0100 Subject: [PATCH 2/5] e2e/storage: drivers determine the claim size Different drivers support different volume sizes. Some have certain minimum sizes, some maximum sizes. Instead of hard-coding some kind of default into the testsuites, now each driver that supports dynamic provisioning has to provide the size. --- test/e2e/storage/drivers/base.go | 5 +++++ test/e2e/storage/drivers/csi.go | 12 +++++++++++ test/e2e/storage/drivers/in_tree.go | 24 +++++++++++++++++++++ test/e2e/storage/testsuites/base.go | 2 +- test/e2e/storage/testsuites/provisioning.go | 2 +- test/e2e/storage/testsuites/volumemode.go | 2 +- 6 files changed, 44 insertions(+), 3 deletions(-) diff --git a/test/e2e/storage/drivers/base.go b/test/e2e/storage/drivers/base.go index 95d6b263c33..b40ce8dcaa2 100644 --- a/test/e2e/storage/drivers/base.go +++ b/test/e2e/storage/drivers/base.go @@ -74,6 +74,11 @@ type DynamicPVTestDriver interface { // It will set fsType to the StorageClass, if TestDriver supports it. // It will return nil, if the TestDriver doesn't support it. GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass + + // GetClaimSize returns the size of the volume that is to be provisioned ("5Gi", "1Mi"). + // The size must be chosen so that the resulting volume is large enough for all + // enabled tests and within the range supported by the underlying storage. + GetClaimSize() string } // Capability represents a feature that a volume plugin supports diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 6bedcb71813..247bdc507f4 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -103,6 +103,10 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *stor return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (h *hostpathCSIDriver) GetClaimSize() string { + return "5Gi" +} + func (h *hostpathCSIDriver) CreateDriver() { By(fmt.Sprintf("deploying %s driver", h.driverInfo.Name)) f := h.driverInfo.Framework @@ -205,6 +209,10 @@ func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storage return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (g *gcePDCSIDriver) GetClaimSize() string { + return "5Gi" +} + func (g *gcePDCSIDriver) CreateDriver() { By("deploying csi gce-pd driver") // It would be safer to rename the gcePD driver, but that @@ -293,6 +301,10 @@ func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(fsType string) return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (g *gcePDExternalCSIDriver) GetClaimSize() string { + return "5Gi" +} + func (g *gcePDExternalCSIDriver) CreateDriver() { } diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index cc73bd81a4c..67f0abc854c 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -138,6 +138,10 @@ func (n *nfsDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.St return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (n *nfsDriver) GetClaimSize() string { + return "5Gi" +} + func (n *nfsDriver) CreateDriver() { f := n.driverInfo.Framework cs := f.ClientSet @@ -1034,6 +1038,10 @@ func (c *cinderDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1 return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (c *cinderDriver) GetClaimSize() string { + return "5Gi" +} + func (c *cinderDriver) CreateDriver() { } @@ -1196,6 +1204,10 @@ func (g *gcePdDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1. return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (h *gcePdDriver) GetClaimSize() string { + return "5Gi" +} + func (g *gcePdDriver) CreateDriver() { } @@ -1318,6 +1330,10 @@ func (v *vSphereDriver) GetDynamicProvisionStorageClass(fsType string) *storagev return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (v *vSphereDriver) GetClaimSize() string { + return "5Gi" +} + func (v *vSphereDriver) CreateDriver() { } @@ -1436,6 +1452,10 @@ func (a *azureDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1. return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (a *azureDriver) GetClaimSize() string { + return "5Gi" +} + func (a *azureDriver) CreateDriver() { } @@ -1543,6 +1563,10 @@ func (a *awsDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.St return getStorageClass(provisioner, parameters, nil, ns, suffix) } +func (a *awsDriver) GetClaimSize() string { + return "5Gi" +} + func (a *awsDriver) CreateDriver() { } diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 8d4f084989f..b7555470342 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -164,7 +164,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat case testpatterns.DynamicPV: framework.Logf("Creating resource for dynamic PV") if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { - claimSize := "5Gi" + claimSize := dDriver.GetClaimSize() r.sc = dDriver.GetDynamicProvisionStorageClass(fsType) By("creating a StorageClass " + r.sc.Name) diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index ba9f8d025b3..9315bbac84e 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -151,7 +151,7 @@ func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, patt framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name) } p.driver = driver - p.claimSize = "5Gi" + p.claimSize = dDriver.GetClaimSize() p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name) p.pvc.Spec.StorageClassName = &p.sc.Name framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc) diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index 94bef7508d9..b2f113cd037 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -196,7 +196,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter } s.sc.VolumeBindingMode = &volBindMode - claimSize := "5Gi" + claimSize := dDriver.GetClaimSize() s.pvc = getClaim(claimSize, ns.Name) s.pvc.Spec.StorageClassName = &s.sc.Name s.pvc.Spec.VolumeMode = &volMode From dfcf2ca345d3830f1e48d8338a753ba23a77eb22 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 9 Nov 2018 11:15:16 +0100 Subject: [PATCH 3/5] e2e/storage: separate generic testsuites from specific drivers This makes it possible to use the testsuites package out-of-tree without pulling in unnecessary dependencies and code (in test/e2e/storage/vsphere) that defines tests that are not wanted in a custom test suite. --- test/e2e/storage/csi_volumes.go | 14 +- test/e2e/storage/drivers/csi.go | 74 ++-- test/e2e/storage/drivers/in_tree.go | 323 +++++++++--------- test/e2e/storage/in_tree_volumes.go | 6 +- test/e2e/storage/testsuites/base.go | 35 +- .../storage/testsuites/driveroperations.go | 114 +++++++ test/e2e/storage/testsuites/provisioning.go | 19 +- test/e2e/storage/testsuites/subpath.go | 11 +- .../base.go => testsuites/testdriver.go} | 91 +---- test/e2e/storage/testsuites/volume_io.go | 7 +- test/e2e/storage/testsuites/volumemode.go | 25 +- test/e2e/storage/testsuites/volumes.go | 15 +- 12 files changed, 378 insertions(+), 356 deletions(-) create mode 100644 test/e2e/storage/testsuites/driveroperations.go rename test/e2e/storage/{drivers/base.go => testsuites/testdriver.go} (65%) diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index e614f4e9513..47f98c60618 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -43,7 +43,7 @@ import ( ) // List of testDrivers to be executed in below loop -var csiTestDrivers = []func() drivers.TestDriver{ +var csiTestDrivers = []func() testsuites.TestDriver{ drivers.InitHostPathCSIDriver, drivers.InitGcePDCSIDriver, drivers.InitGcePDExternalCSIDriver, @@ -126,12 +126,12 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { for _, initDriver := range csiTestDrivers { curDriver := initDriver() - Context(drivers.GetDriverNameWithFeatureTags(curDriver), func() { + Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() { driver := curDriver BeforeEach(func() { // setupDriver - drivers.SetCommonDriverParameters(driver, f, config) + testsuites.SetCommonDriverParameters(driver, f, config) driver.CreateDriver() }) @@ -149,14 +149,14 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { var ( cs clientset.Interface csics csiclient.Interface - driver drivers.TestDriver + driver testsuites.TestDriver ) BeforeEach(func() { cs = f.ClientSet csics = f.CSIClientSet driver = drivers.InitHostPathCSIDriver() - drivers.SetCommonDriverParameters(driver, f, config) + testsuites.SetCommonDriverParameters(driver, f, config) driver.CreateDriver() }) @@ -193,7 +193,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { test := t It(test.name, func() { if test.driverExists { - csiDriver := createCSIDriver(csics, drivers.GetUniqueDriverName(driver), test.driverAttachable) + csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), test.driverAttachable) if csiDriver != nil { defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil) } @@ -201,7 +201,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { By("Creating pod") var sc *storagev1.StorageClass - if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { + if dDriver, ok := driver.(testsuites.DynamicPVTestDriver); ok { sc = dDriver.GetDynamicProvisionStorageClass("") } nodeName := driver.GetDriverInfo().Config.ClientNodeName diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 247bdc507f4..62187a0f6c8 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -44,38 +44,39 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testpatterns" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" ) // hostpathCSI type hostpathCSIDriver struct { cleanup func() - driverInfo DriverInfo + driverInfo testsuites.DriverInfo manifests []string } -func initHostPathCSIDriver(name string, manifests ...string) TestDriver { +func initHostPathCSIDriver(name string, manifests ...string) testsuites.TestDriver { return &hostpathCSIDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: name, FeatureTag: "", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType ), - Capabilities: map[Capability]bool{ - CapPersistence: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, }, }, manifests: manifests, } } -var _ TestDriver = &hostpathCSIDriver{} -var _ DynamicPVTestDriver = &hostpathCSIDriver{} +var _ testsuites.TestDriver = &hostpathCSIDriver{} +var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{} // InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface -func InitHostPathCSIDriver() TestDriver { +func InitHostPathCSIDriver() testsuites.TestDriver { return initHostPathCSIDriver("csi-hostpath", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", @@ -87,7 +88,7 @@ func InitHostPathCSIDriver() TestDriver { ) } -func (h *hostpathCSIDriver) GetDriverInfo() *DriverInfo { +func (h *hostpathCSIDriver) GetDriverInfo() *testsuites.DriverInfo { return &h.driverInfo } @@ -95,12 +96,12 @@ func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern } func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { - provisioner := GetUniqueDriverName(h) + provisioner := testsuites.GetUniqueDriverName(h) parameters := map[string]string{} ns := h.driverInfo.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", provisioner) - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (h *hostpathCSIDriver) GetClaimSize() string { @@ -122,7 +123,7 @@ func (h *hostpathCSIDriver) CreateDriver() { // settings are ignored for this test. We could patch the image definitions. o := utils.PatchCSIOptions{ OldDriverName: h.driverInfo.Name, - NewDriverName: GetUniqueDriverName(h), + NewDriverName: testsuites.GetUniqueDriverName(h), DriverContainerName: "hostpath", ProvisionerContainerName: "csi-provisioner", NodeName: h.driverInfo.Config.ServerNodeName, @@ -145,7 +146,7 @@ func (h *hostpathCSIDriver) CleanupDriver() { } // InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests. -func InitHostPathV0CSIDriver() TestDriver { +func InitHostPathV0CSIDriver() testsuites.TestDriver { return initHostPathCSIDriver("csi-hostpath-v0", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", @@ -160,16 +161,16 @@ func InitHostPathV0CSIDriver() TestDriver { // gce-pd type gcePDCSIDriver struct { cleanup func() - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } -var _ TestDriver = &gcePDCSIDriver{} -var _ DynamicPVTestDriver = &gcePDCSIDriver{} +var _ testsuites.TestDriver = &gcePDCSIDriver{} +var _ testsuites.DynamicPVTestDriver = &gcePDCSIDriver{} // InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface -func InitGcePDCSIDriver() TestDriver { +func InitGcePDCSIDriver() testsuites.TestDriver { return &gcePDCSIDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "pd.csi.storage.gke.io", FeatureTag: "[Serial]", MaxFileSize: testpatterns.FileSizeMedium, @@ -180,16 +181,16 @@ func InitGcePDCSIDriver() TestDriver { "ext4", "xfs", ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapExec: true, }, }, } } -func (g *gcePDCSIDriver) GetDriverInfo() *DriverInfo { +func (g *gcePDCSIDriver) GetDriverInfo() *testsuites.DriverInfo { return &g.driverInfo } @@ -206,7 +207,7 @@ func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storage parameters := map[string]string{"type": "pd-standard"} - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (g *gcePDCSIDriver) GetClaimSize() string { @@ -223,7 +224,7 @@ func (g *gcePDCSIDriver) CreateDriver() { // These are the options which would have to be used: // o := utils.PatchCSIOptions{ // OldDriverName: g.driverInfo.Name, - // NewDriverName: GetUniqueDriverName(g), + // NewDriverName: testsuites.GetUniqueDriverName(g), // DriverContainerName: "gce-driver", // ProvisionerContainerName: "csi-external-provisioner", // } @@ -252,17 +253,18 @@ func (g *gcePDCSIDriver) CleanupDriver() { // gcePd-external type gcePDExternalCSIDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } -var _ TestDriver = &gcePDExternalCSIDriver{} -var _ DynamicPVTestDriver = &gcePDExternalCSIDriver{} +var _ testsuites.TestDriver = &gcePDExternalCSIDriver{} +var _ testsuites.DynamicPVTestDriver = &gcePDExternalCSIDriver{} // InitGcePDExternalCSIDriver returns gcePDExternalCSIDriver that implements TestDriver interface -func InitGcePDExternalCSIDriver() TestDriver { +func InitGcePDExternalCSIDriver() testsuites.TestDriver { return &gcePDExternalCSIDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "pd.csi.storage.gke.io", + // TODO(#70258): this is temporary until we can figure out how to make e2e tests a library FeatureTag: "[Feature: gcePD-external]", MaxFileSize: testpatterns.FileSizeMedium, @@ -273,16 +275,16 @@ func InitGcePDExternalCSIDriver() TestDriver { "ext4", "xfs", ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapExec: true, }, }, } } -func (g *gcePDExternalCSIDriver) GetDriverInfo() *DriverInfo { +func (g *gcePDExternalCSIDriver) GetDriverInfo() *testsuites.DriverInfo { return &g.driverInfo } @@ -298,7 +300,7 @@ func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(fsType string) parameters := map[string]string{"type": "pd-standard"} - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (g *gcePDExternalCSIDriver) GetClaimSize() string { diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 67f0abc854c..d121f3e4fa9 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -55,6 +55,7 @@ import ( kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testpatterns" + "k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/utils" vspheretest "k8s.io/kubernetes/test/e2e/storage/vsphere" imageutils "k8s.io/kubernetes/test/utils/image" @@ -65,7 +66,7 @@ type nfsDriver struct { externalProvisionerPod *v1.Pod externalPluginName string - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type nfsTestResource struct { @@ -73,16 +74,16 @@ type nfsTestResource struct { serverPod *v1.Pod } -var _ TestDriver = &nfsDriver{} -var _ PreprovisionedVolumeTestDriver = &nfsDriver{} -var _ InlineVolumeTestDriver = &nfsDriver{} -var _ PreprovisionedPVTestDriver = &nfsDriver{} -var _ DynamicPVTestDriver = &nfsDriver{} +var _ testsuites.TestDriver = &nfsDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &nfsDriver{} +var _ testsuites.InlineVolumeTestDriver = &nfsDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &nfsDriver{} +var _ testsuites.DynamicPVTestDriver = &nfsDriver{} // InitNFSDriver returns nfsDriver that implements TestDriver interface -func InitNFSDriver() TestDriver { +func InitNFSDriver() testsuites.TestDriver { return &nfsDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "nfs", MaxFileSize: testpatterns.FileSizeLarge, SupportedFsType: sets.NewString( @@ -90,15 +91,15 @@ func InitNFSDriver() TestDriver { ), SupportedMountOption: sets.NewString("proto=tcp", "relatime"), RequiredMountOption: sets.NewString("vers=4.1"), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapExec: true, }, }, } } -func (n *nfsDriver) GetDriverInfo() *DriverInfo { +func (n *nfsDriver) GetDriverInfo() *testsuites.DriverInfo { return &n.driverInfo } @@ -135,7 +136,7 @@ func (n *nfsDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.St ns := n.driverInfo.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", n.driverInfo.Name) - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (n *nfsDriver) GetClaimSize() string { @@ -218,7 +219,7 @@ func (n *nfsDriver) DeleteVolume(volType testpatterns.TestVolType, testResource // Gluster type glusterFSDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type glusterTestResource struct { @@ -226,29 +227,29 @@ type glusterTestResource struct { serverPod *v1.Pod } -var _ TestDriver = &glusterFSDriver{} -var _ PreprovisionedVolumeTestDriver = &glusterFSDriver{} -var _ InlineVolumeTestDriver = &glusterFSDriver{} -var _ PreprovisionedPVTestDriver = &glusterFSDriver{} +var _ testsuites.TestDriver = &glusterFSDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &glusterFSDriver{} +var _ testsuites.InlineVolumeTestDriver = &glusterFSDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &glusterFSDriver{} // InitGlusterFSDriver returns glusterFSDriver that implements TestDriver interface -func InitGlusterFSDriver() TestDriver { +func InitGlusterFSDriver() testsuites.TestDriver { return &glusterFSDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "gluster", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapExec: true, }, }, } } -func (g *glusterFSDriver) GetDriverInfo() *DriverInfo { +func (g *glusterFSDriver) GetDriverInfo() *testsuites.DriverInfo { return &g.driverInfo } @@ -336,22 +337,22 @@ func (g *glusterFSDriver) DeleteVolume(volType testpatterns.TestVolType, testRes // iSCSI // The iscsiadm utility and iscsi target kernel modules must be installed on all nodes. type iSCSIDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type iSCSITestResource struct { serverPod *v1.Pod serverIP string } -var _ TestDriver = &iSCSIDriver{} -var _ PreprovisionedVolumeTestDriver = &iSCSIDriver{} -var _ InlineVolumeTestDriver = &iSCSIDriver{} -var _ PreprovisionedPVTestDriver = &iSCSIDriver{} +var _ testsuites.TestDriver = &iSCSIDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &iSCSIDriver{} +var _ testsuites.InlineVolumeTestDriver = &iSCSIDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &iSCSIDriver{} // InitISCSIDriver returns iSCSIDriver that implements TestDriver interface -func InitISCSIDriver() TestDriver { +func InitISCSIDriver() testsuites.TestDriver { return &iSCSIDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "iscsi", FeatureTag: "[Feature:Volumes]", MaxFileSize: testpatterns.FileSizeMedium, @@ -362,17 +363,17 @@ func InitISCSIDriver() TestDriver { //"ext3", "ext4", ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapBlock: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapBlock: true, + testsuites.CapExec: true, }, }, } } -func (i *iSCSIDriver) GetDriverInfo() *DriverInfo { +func (i *iSCSIDriver) GetDriverInfo() *testsuites.DriverInfo { return &i.driverInfo } @@ -446,7 +447,7 @@ func (i *iSCSIDriver) DeleteVolume(volType testpatterns.TestVolType, testResourc // Ceph RBD type rbdDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type rbdTestResource struct { @@ -455,15 +456,15 @@ type rbdTestResource struct { secret *v1.Secret } -var _ TestDriver = &rbdDriver{} -var _ PreprovisionedVolumeTestDriver = &rbdDriver{} -var _ InlineVolumeTestDriver = &rbdDriver{} -var _ PreprovisionedPVTestDriver = &rbdDriver{} +var _ testsuites.TestDriver = &rbdDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &rbdDriver{} +var _ testsuites.InlineVolumeTestDriver = &rbdDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &rbdDriver{} // InitRbdDriver returns rbdDriver that implements TestDriver interface -func InitRbdDriver() TestDriver { +func InitRbdDriver() testsuites.TestDriver { return &rbdDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "rbd", FeatureTag: "[Feature:Volumes]", MaxFileSize: testpatterns.FileSizeMedium, @@ -474,17 +475,17 @@ func InitRbdDriver() TestDriver { //"ext3", "ext4", ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapBlock: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapBlock: true, + testsuites.CapExec: true, }, }, } } -func (r *rbdDriver) GetDriverInfo() *DriverInfo { +func (r *rbdDriver) GetDriverInfo() *testsuites.DriverInfo { return &r.driverInfo } @@ -574,7 +575,7 @@ type cephFSDriver struct { serverPod *v1.Pod secret *v1.Secret - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type cephTestResource struct { @@ -583,30 +584,30 @@ type cephTestResource struct { secret *v1.Secret } -var _ TestDriver = &cephFSDriver{} -var _ PreprovisionedVolumeTestDriver = &cephFSDriver{} -var _ InlineVolumeTestDriver = &cephFSDriver{} -var _ PreprovisionedPVTestDriver = &cephFSDriver{} +var _ testsuites.TestDriver = &cephFSDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &cephFSDriver{} +var _ testsuites.InlineVolumeTestDriver = &cephFSDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &cephFSDriver{} // InitCephFSDriver returns cephFSDriver that implements TestDriver interface -func InitCephFSDriver() TestDriver { +func InitCephFSDriver() testsuites.TestDriver { return &cephFSDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "ceph", FeatureTag: "[Feature:Volumes]", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapExec: true, }, }, } } -func (c *cephFSDriver) GetDriverInfo() *DriverInfo { +func (c *cephFSDriver) GetDriverInfo() *testsuites.DriverInfo { return &c.driverInfo } @@ -682,30 +683,30 @@ func (c *cephFSDriver) DeleteVolume(volType testpatterns.TestVolType, testResour type hostPathDriver struct { node v1.Node - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } -var _ TestDriver = &hostPathDriver{} -var _ PreprovisionedVolumeTestDriver = &hostPathDriver{} -var _ InlineVolumeTestDriver = &hostPathDriver{} +var _ testsuites.TestDriver = &hostPathDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &hostPathDriver{} +var _ testsuites.InlineVolumeTestDriver = &hostPathDriver{} // InitHostPathDriver returns hostPathDriver that implements TestDriver interface -func InitHostPathDriver() TestDriver { +func InitHostPathDriver() testsuites.TestDriver { return &hostPathDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "hostPath", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType ), - Capabilities: map[Capability]bool{ - CapPersistence: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, }, }, } } -func (h *hostPathDriver) GetDriverInfo() *DriverInfo { +func (h *hostPathDriver) GetDriverInfo() *testsuites.DriverInfo { return &h.driverInfo } @@ -748,7 +749,7 @@ func (h *hostPathDriver) DeleteVolume(volType testpatterns.TestVolType, testReso type hostPathSymlinkDriver struct { node v1.Node - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type hostPathSymlinkTestResource struct { @@ -757,27 +758,27 @@ type hostPathSymlinkTestResource struct { prepPod *v1.Pod } -var _ TestDriver = &hostPathSymlinkDriver{} -var _ PreprovisionedVolumeTestDriver = &hostPathSymlinkDriver{} -var _ InlineVolumeTestDriver = &hostPathSymlinkDriver{} +var _ testsuites.TestDriver = &hostPathSymlinkDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &hostPathSymlinkDriver{} +var _ testsuites.InlineVolumeTestDriver = &hostPathSymlinkDriver{} // InitHostPathSymlinkDriver returns hostPathSymlinkDriver that implements TestDriver interface -func InitHostPathSymlinkDriver() TestDriver { +func InitHostPathSymlinkDriver() testsuites.TestDriver { return &hostPathSymlinkDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "hostPathSymlink", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType ), - Capabilities: map[Capability]bool{ - CapPersistence: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, }, }, } } -func (h *hostPathSymlinkDriver) GetDriverInfo() *DriverInfo { +func (h *hostPathSymlinkDriver) GetDriverInfo() *testsuites.DriverInfo { return &h.driverInfo } @@ -894,30 +895,30 @@ func (h *hostPathSymlinkDriver) DeleteVolume(volType testpatterns.TestVolType, t // emptydir type emptydirDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } -var _ TestDriver = &emptydirDriver{} -var _ PreprovisionedVolumeTestDriver = &emptydirDriver{} -var _ InlineVolumeTestDriver = &emptydirDriver{} +var _ testsuites.TestDriver = &emptydirDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &emptydirDriver{} +var _ testsuites.InlineVolumeTestDriver = &emptydirDriver{} // InitEmptydirDriver returns emptydirDriver that implements TestDriver interface -func InitEmptydirDriver() TestDriver { +func InitEmptydirDriver() testsuites.TestDriver { return &emptydirDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "emptydir", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType ), - Capabilities: map[Capability]bool{ - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapExec: true, }, }, } } -func (e *emptydirDriver) GetDriverInfo() *DriverInfo { +func (e *emptydirDriver) GetDriverInfo() *testsuites.DriverInfo { return &e.driverInfo } @@ -953,7 +954,7 @@ func (e *emptydirDriver) CleanupDriver() { // and that the usual OpenStack authentication env. variables are set // (OS_USERNAME, OS_PASSWORD, OS_TENANT_NAME at least). type cinderDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type cinderTestResource struct { @@ -961,32 +962,32 @@ type cinderTestResource struct { volumeID string } -var _ TestDriver = &cinderDriver{} -var _ PreprovisionedVolumeTestDriver = &cinderDriver{} -var _ InlineVolumeTestDriver = &cinderDriver{} -var _ PreprovisionedPVTestDriver = &cinderDriver{} -var _ DynamicPVTestDriver = &cinderDriver{} +var _ testsuites.TestDriver = &cinderDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &cinderDriver{} +var _ testsuites.InlineVolumeTestDriver = &cinderDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{} +var _ testsuites.DynamicPVTestDriver = &cinderDriver{} // InitCinderDriver returns cinderDriver that implements TestDriver interface -func InitCinderDriver() TestDriver { +func InitCinderDriver() testsuites.TestDriver { return &cinderDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "cinder", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType "ext3", ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapExec: true, }, }, } } -func (c *cinderDriver) GetDriverInfo() *DriverInfo { +func (c *cinderDriver) GetDriverInfo() *testsuites.DriverInfo { return &c.driverInfo } @@ -1035,7 +1036,7 @@ func (c *cinderDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1 ns := c.driverInfo.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", c.driverInfo.Name) - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (c *cinderDriver) GetClaimSize() string { @@ -1114,23 +1115,23 @@ func deleteCinderVolume(name string) error { // GCE type gcePdDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type gcePdTestResource struct { volumeName string } -var _ TestDriver = &gcePdDriver{} -var _ PreprovisionedVolumeTestDriver = &gcePdDriver{} -var _ InlineVolumeTestDriver = &gcePdDriver{} -var _ PreprovisionedPVTestDriver = &gcePdDriver{} -var _ DynamicPVTestDriver = &gcePdDriver{} +var _ testsuites.TestDriver = &gcePdDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &gcePdDriver{} +var _ testsuites.InlineVolumeTestDriver = &gcePdDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{} +var _ testsuites.DynamicPVTestDriver = &gcePdDriver{} // InitGceDriver returns gcePdDriver that implements TestDriver interface -func InitGcePdDriver() TestDriver { +func InitGcePdDriver() testsuites.TestDriver { return &gcePdDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "gcepd", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( @@ -1141,17 +1142,17 @@ func InitGcePdDriver() TestDriver { "xfs", ), SupportedMountOption: sets.NewString("debug", "nouid32"), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapBlock: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapBlock: true, + testsuites.CapExec: true, }, }, } } -func (g *gcePdDriver) GetDriverInfo() *DriverInfo { +func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo { return &g.driverInfo } @@ -1201,7 +1202,7 @@ func (g *gcePdDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1. ns := g.driverInfo.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (h *gcePdDriver) GetClaimSize() string { @@ -1238,7 +1239,7 @@ func (g *gcePdDriver) DeleteVolume(volType testpatterns.TestVolType, testResourc // vSphere type vSphereDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type vSphereTestResource struct { @@ -1246,31 +1247,31 @@ type vSphereTestResource struct { nodeInfo *vspheretest.NodeInfo } -var _ TestDriver = &vSphereDriver{} -var _ PreprovisionedVolumeTestDriver = &vSphereDriver{} -var _ InlineVolumeTestDriver = &vSphereDriver{} -var _ PreprovisionedPVTestDriver = &vSphereDriver{} -var _ DynamicPVTestDriver = &vSphereDriver{} +var _ testsuites.TestDriver = &vSphereDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &vSphereDriver{} +var _ testsuites.InlineVolumeTestDriver = &vSphereDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &vSphereDriver{} +var _ testsuites.DynamicPVTestDriver = &vSphereDriver{} // InitVSphereDriver returns vSphereDriver that implements TestDriver interface -func InitVSphereDriver() TestDriver { +func InitVSphereDriver() testsuites.TestDriver { return &vSphereDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "vSphere", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType "ext4", ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapExec: true, }, }, } } -func (v *vSphereDriver) GetDriverInfo() *DriverInfo { +func (v *vSphereDriver) GetDriverInfo() *testsuites.DriverInfo { return &v.driverInfo } @@ -1327,7 +1328,7 @@ func (v *vSphereDriver) GetDynamicProvisionStorageClass(fsType string) *storagev ns := v.driverInfo.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", v.driverInfo.Name) - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (v *vSphereDriver) GetClaimSize() string { @@ -1361,40 +1362,40 @@ func (v *vSphereDriver) DeleteVolume(volType testpatterns.TestVolType, testResou // Azure type azureDriver struct { - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } type azureTestResource struct { volumeName string } -var _ TestDriver = &azureDriver{} -var _ PreprovisionedVolumeTestDriver = &azureDriver{} -var _ InlineVolumeTestDriver = &azureDriver{} -var _ PreprovisionedPVTestDriver = &azureDriver{} -var _ DynamicPVTestDriver = &azureDriver{} +var _ testsuites.TestDriver = &azureDriver{} +var _ testsuites.PreprovisionedVolumeTestDriver = &azureDriver{} +var _ testsuites.InlineVolumeTestDriver = &azureDriver{} +var _ testsuites.PreprovisionedPVTestDriver = &azureDriver{} +var _ testsuites.DynamicPVTestDriver = &azureDriver{} // InitAzureDriver returns azureDriver that implements TestDriver interface -func InitAzureDriver() TestDriver { +func InitAzureDriver() testsuites.TestDriver { return &azureDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "azure", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( "", // Default fsType "ext4", ), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapBlock: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapBlock: true, + testsuites.CapExec: true, }, }, } } -func (a *azureDriver) GetDriverInfo() *DriverInfo { +func (a *azureDriver) GetDriverInfo() *testsuites.DriverInfo { return &a.driverInfo } @@ -1449,7 +1450,7 @@ func (a *azureDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1. ns := a.driverInfo.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name) - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (a *azureDriver) GetClaimSize() string { @@ -1482,21 +1483,21 @@ func (a *azureDriver) DeleteVolume(volType testpatterns.TestVolType, testResourc type awsDriver struct { volumeName string - driverInfo DriverInfo + driverInfo testsuites.DriverInfo } -var _ TestDriver = &awsDriver{} +var _ testsuites.TestDriver = &awsDriver{} // TODO: Fix authorization error in attach operation and uncomment below -//var _ PreprovisionedVolumeTestDriver = &awsDriver{} -//var _ InlineVolumeTestDriver = &awsDriver{} -//var _ PreprovisionedPVTestDriver = &awsDriver{} -var _ DynamicPVTestDriver = &awsDriver{} +//var _ testsuites.PreprovisionedVolumeTestDriver = &awsDriver{} +//var _ testsuites.InlineVolumeTestDriver = &awsDriver{} +//var _ testsuites.PreprovisionedPVTestDriver = &awsDriver{} +var _ testsuites.DynamicPVTestDriver = &awsDriver{} // InitAwsDriver returns awsDriver that implements TestDriver interface -func InitAwsDriver() TestDriver { +func InitAwsDriver() testsuites.TestDriver { return &awsDriver{ - driverInfo: DriverInfo{ + driverInfo: testsuites.DriverInfo{ Name: "aws", MaxFileSize: testpatterns.FileSizeMedium, SupportedFsType: sets.NewString( @@ -1504,17 +1505,17 @@ func InitAwsDriver() TestDriver { "ext3", ), SupportedMountOption: sets.NewString("debug", "nouid32"), - Capabilities: map[Capability]bool{ - CapPersistence: true, - CapFsGroup: true, - CapBlock: true, - CapExec: true, + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: true, + testsuites.CapFsGroup: true, + testsuites.CapBlock: true, + testsuites.CapExec: true, }, }, } } -func (a *awsDriver) GetDriverInfo() *DriverInfo { +func (a *awsDriver) GetDriverInfo() *testsuites.DriverInfo { return &a.driverInfo } @@ -1560,7 +1561,7 @@ func (a *awsDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.St ns := a.driverInfo.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name) - return getStorageClass(provisioner, parameters, nil, ns, suffix) + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } func (a *awsDriver) GetClaimSize() string { diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 603876150b5..11c9e73459d 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -27,7 +27,7 @@ import ( ) // List of testDrivers to be executed in below loop -var testDrivers = []func() drivers.TestDriver{ +var testDrivers = []func() testsuites.TestDriver{ drivers.InitNFSDriver, drivers.InitGlusterFSDriver, drivers.InitISCSIDriver, @@ -75,12 +75,12 @@ var _ = utils.SIGDescribe("In-tree Volumes", func() { for _, initDriver := range testDrivers { curDriver := initDriver() - Context(drivers.GetDriverNameWithFeatureTags(curDriver), func() { + Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() { driver := curDriver BeforeEach(func() { // setupDriver - drivers.SetCommonDriverParameters(driver, f, config) + testsuites.SetCommonDriverParameters(driver, f, config) driver.CreateDriver() }) diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index b7555470342..c257b849620 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -30,7 +30,6 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/testpatterns" ) @@ -39,9 +38,9 @@ type TestSuite interface { // getTestSuiteInfo returns the TestSuiteInfo for this TestSuite getTestSuiteInfo() TestSuiteInfo // skipUnsupportedTest skips the test if this TestSuite is not suitable to be tested with the combination of TestPattern and TestDriver - skipUnsupportedTest(testpatterns.TestPattern, drivers.TestDriver) + skipUnsupportedTest(testpatterns.TestPattern, TestDriver) // execTest executes test of the testpattern for the driver - execTest(drivers.TestDriver, testpatterns.TestPattern) + execTest(TestDriver, testpatterns.TestPattern) } // TestSuiteInfo represents a set of parameters for TestSuite @@ -55,9 +54,9 @@ type TestSuiteInfo struct { type TestResource interface { // setupResource sets up test resources to be used for the tests with the // combination of TestDriver and TestPattern - setupResource(drivers.TestDriver, testpatterns.TestPattern) + setupResource(TestDriver, testpatterns.TestPattern) // cleanupResource clean up the test resources created in SetupResource - cleanupResource(drivers.TestDriver, testpatterns.TestPattern) + cleanupResource(TestDriver, testpatterns.TestPattern) } func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string { @@ -66,7 +65,7 @@ func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string { } // RunTestSuite runs all testpatterns of all testSuites for a driver -func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) { +func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) { for _, testSuiteInit := range tsInits { suite := testSuiteInit() patterns := tunePatternFunc(suite.getTestSuiteInfo().testPatterns) @@ -84,18 +83,18 @@ func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, dri // 2. Check if fsType is supported by driver // 3. Check with driver specific logic // 4. Check with testSuite specific logic -func skipUnsupportedTest(suite TestSuite, driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func skipUnsupportedTest(suite TestSuite, driver TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() // 1. Check if Whether volType is supported by driver from its interface var isSupported bool switch pattern.VolType { case testpatterns.InlineVolume: - _, isSupported = driver.(drivers.InlineVolumeTestDriver) + _, isSupported = driver.(InlineVolumeTestDriver) case testpatterns.PreprovisionedPV: - _, isSupported = driver.(drivers.PreprovisionedPVTestDriver) + _, isSupported = driver.(PreprovisionedPVTestDriver) case testpatterns.DynamicPV: - _, isSupported = driver.(drivers.DynamicPVTestDriver) + _, isSupported = driver.(DynamicPVTestDriver) default: isSupported = false } @@ -121,7 +120,7 @@ func skipUnsupportedTest(suite TestSuite, driver drivers.TestDriver, pattern tes // See volume_io.go or volumes.go in test/e2e/storage/testsuites/ for how to use this resource. // Also, see subpath.go in the same directory for how to extend and use it. type genericVolumeTestResource struct { - driver drivers.TestDriver + driver TestDriver volType string volSource *v1.VolumeSource pvc *v1.PersistentVolumeClaim @@ -134,7 +133,7 @@ type genericVolumeTestResource struct { var _ TestResource = &genericVolumeTestResource{} // setupResource sets up genericVolumeTestResource -func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (r *genericVolumeTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { r.driver = driver dInfo := driver.GetDriverInfo() f := dInfo.Framework @@ -143,18 +142,18 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat volType := pattern.VolType // Create volume for pre-provisioned volume tests - r.driverTestResource = drivers.CreateVolume(driver, volType) + r.driverTestResource = CreateVolume(driver, volType) switch volType { case testpatterns.InlineVolume: framework.Logf("Creating resource for inline volume") - if iDriver, ok := driver.(drivers.InlineVolumeTestDriver); ok { + if iDriver, ok := driver.(InlineVolumeTestDriver); ok { r.volSource = iDriver.GetVolumeSource(false, fsType, r.driverTestResource) r.volType = dInfo.Name } case testpatterns.PreprovisionedPV: framework.Logf("Creating resource for pre-provisioned PV") - if pDriver, ok := driver.(drivers.PreprovisionedPVTestDriver); ok { + if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok { pvSource := pDriver.GetPersistentVolumeSource(false, fsType, r.driverTestResource) if pvSource != nil { r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, false) @@ -163,7 +162,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat } case testpatterns.DynamicPV: framework.Logf("Creating resource for dynamic PV") - if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { + if dDriver, ok := driver.(DynamicPVTestDriver); ok { claimSize := dDriver.GetClaimSize() r.sc = dDriver.GetDynamicProvisionStorageClass(fsType) @@ -188,7 +187,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat } // cleanupResource cleans up genericVolumeTestResource -func (r *genericVolumeTestResource) cleanupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (r *genericVolumeTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() f := dInfo.Framework volType := pattern.VolType @@ -222,7 +221,7 @@ func (r *genericVolumeTestResource) cleanupResource(driver drivers.TestDriver, p } // Cleanup volume for pre-provisioned volume tests - drivers.DeleteVolume(driver, volType, r.driverTestResource) + DeleteVolume(driver, volType, r.driverTestResource) } func createVolumeSourceWithPVCPV( diff --git a/test/e2e/storage/testsuites/driveroperations.go b/test/e2e/storage/testsuites/driveroperations.go new file mode 100644 index 00000000000..41c2d3eb22f --- /dev/null +++ b/test/e2e/storage/testsuites/driveroperations.go @@ -0,0 +1,114 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "fmt" + + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/testpatterns" +) + +// GetDriverNameWithFeatureTags returns driver name with feature tags +// For example) +// - [Driver: nfs] +// - [Driver: rbd][Feature:Volumes] +func GetDriverNameWithFeatureTags(driver TestDriver) string { + dInfo := driver.GetDriverInfo() + + return fmt.Sprintf("[Driver: %s]%s", dInfo.Name, dInfo.FeatureTag) +} + +// CreateVolume creates volume for test unless dynamicPV test +func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface{} { + switch volType { + case testpatterns.InlineVolume: + fallthrough + case testpatterns.PreprovisionedPV: + if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok { + return pDriver.CreateVolume(volType) + } + case testpatterns.DynamicPV: + // No need to create volume + default: + framework.Failf("Invalid volType specified: %v", volType) + } + return nil +} + +// DeleteVolume deletes volume for test unless dynamicPV test +func DeleteVolume(driver TestDriver, volType testpatterns.TestVolType, testResource interface{}) { + switch volType { + case testpatterns.InlineVolume: + fallthrough + case testpatterns.PreprovisionedPV: + if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok { + pDriver.DeleteVolume(volType, testResource) + } + case testpatterns.DynamicPV: + // No need to delete volume + default: + framework.Failf("Invalid volType specified: %v", volType) + } +} + +// SetCommonDriverParameters sets a common driver parameters to TestDriver +// This function is intended to be called in BeforeEach() inside test loop. +func SetCommonDriverParameters( + driver TestDriver, + f *framework.Framework, + config framework.VolumeTestConfig, +) { + dInfo := driver.GetDriverInfo() + + dInfo.Framework = f + dInfo.Config = config +} + +// GetStorageClass constructs a new StorageClass instance +// with a unique name that is based on namespace + suffix. +func GetStorageClass( + provisioner string, + parameters map[string]string, + bindingMode *storagev1.VolumeBindingMode, + ns string, + suffix string, +) *storagev1.StorageClass { + if bindingMode == nil { + defaultBindingMode := storagev1.VolumeBindingImmediate + bindingMode = &defaultBindingMode + } + return &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "StorageClass", + }, + ObjectMeta: metav1.ObjectMeta{ + // Name must be unique, so let's base it on namespace name + Name: ns + "-" + suffix, + }, + Provisioner: provisioner, + Parameters: parameters, + VolumeBindingMode: bindingMode, + } +} + +// GetUniqueDriverName returns unique driver name that can be used parallelly in tests +func GetUniqueDriverName(driver TestDriver) string { + return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Framework.UniqueName) +} diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 9315bbac84e..7524af9fc1c 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -31,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/testpatterns" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -74,10 +73,10 @@ func (p *provisioningTestSuite) getTestSuiteInfo() TestSuiteInfo { return p.tsInfo } -func (p *provisioningTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver drivers.TestDriver) { +func (p *provisioningTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { } -func createProvisioningTestInput(driver drivers.TestDriver, pattern testpatterns.TestPattern) (provisioningTestResource, provisioningTestInput) { +func createProvisioningTestInput(driver TestDriver, pattern testpatterns.TestPattern) (provisioningTestResource, provisioningTestInput) { // Setup test resource for driver and testpattern resource := provisioningTestResource{} resource.setupResource(driver, pattern) @@ -100,7 +99,7 @@ func createProvisioningTestInput(driver drivers.TestDriver, pattern testpatterns return resource, input } -func (p *provisioningTestSuite) execTest(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (p *provisioningTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { Context(getTestNameStr(p, pattern), func() { var ( resource provisioningTestResource @@ -132,7 +131,7 @@ func (p *provisioningTestSuite) execTest(driver drivers.TestDriver, pattern test } type provisioningTestResource struct { - driver drivers.TestDriver + driver TestDriver claimSize string sc *storage.StorageClass @@ -141,11 +140,11 @@ type provisioningTestResource struct { var _ TestResource = &provisioningTestResource{} -func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (p *provisioningTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { // Setup provisioningTest resource switch pattern.VolType { case testpatterns.DynamicPV: - if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { + if dDriver, ok := driver.(DynamicPVTestDriver); ok { p.sc = dDriver.GetDynamicProvisionStorageClass("") if p.sc == nil { framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name) @@ -161,7 +160,7 @@ func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, patt } } -func (p *provisioningTestResource) cleanupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (p *provisioningTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { } type provisioningTestInput struct { @@ -169,7 +168,7 @@ type provisioningTestInput struct { cs clientset.Interface pvc *v1.PersistentVolumeClaim sc *storage.StorageClass - dInfo *drivers.DriverInfo + dInfo *DriverInfo } func testProvisioning(input *provisioningTestInput) { @@ -187,7 +186,7 @@ func testProvisioning(input *provisioningTestInput) { }) It("should create and delete block persistent volumes", func() { - if !input.dInfo.Capabilities[drivers.CapBlock] { + if !input.dInfo.Capabilities[CapBlock] { framework.Skipf("Driver %q does not support BlockVolume - skipping", input.dInfo.Name) } block := v1.PersistentVolumeBlock diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index 66e1985c9ce..50daffeaeee 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -72,7 +71,7 @@ func (s *subPathTestSuite) getTestSuiteInfo() TestSuiteInfo { return s.tsInfo } -func (s *subPathTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver drivers.TestDriver) { +func (s *subPathTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { } func createSubPathTestInput(pattern testpatterns.TestPattern, resource subPathTestResource) subPathTestInput { @@ -95,7 +94,7 @@ func createSubPathTestInput(pattern testpatterns.TestPattern, resource subPathTe } } -func (s *subPathTestSuite) execTest(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (s *subPathTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { Context(getTestNameStr(s, pattern), func() { var ( resource subPathTestResource @@ -137,7 +136,7 @@ type subPathTestResource struct { var _ TestResource = &subPathTestResource{} -func (s *subPathTestResource) setupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (s *subPathTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { s.driver = driver dInfo := s.driver.GetDriverInfo() f := dInfo.Framework @@ -150,7 +149,7 @@ func (s *subPathTestResource) setupResource(driver drivers.TestDriver, pattern t // Setup subPath test dependent resource switch volType { case testpatterns.InlineVolume: - if iDriver, ok := driver.(drivers.InlineVolumeTestDriver); ok { + if iDriver, ok := driver.(InlineVolumeTestDriver); ok { s.roVolSource = iDriver.GetVolumeSource(true, fsType, s.genericVolumeTestResource.driverTestResource) } case testpatterns.PreprovisionedPV: @@ -182,7 +181,7 @@ func (s *subPathTestResource) setupResource(driver drivers.TestDriver, pattern t s.formatPod.Spec.NodeSelector = config.NodeSelector } -func (s *subPathTestResource) cleanupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (s *subPathTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() f := dInfo.Framework diff --git a/test/e2e/storage/drivers/base.go b/test/e2e/storage/testsuites/testdriver.go similarity index 65% rename from test/e2e/storage/drivers/base.go rename to test/e2e/storage/testsuites/testdriver.go index b40ce8dcaa2..6af6ba70855 100644 --- a/test/e2e/storage/drivers/base.go +++ b/test/e2e/storage/testsuites/testdriver.go @@ -14,14 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package drivers +package testsuites import ( - "fmt" - "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testpatterns" @@ -108,89 +105,3 @@ type DriverInfo struct { Framework *framework.Framework // Framework for the test Config framework.VolumeTestConfig // VolumeTestConfig for thet test } - -// GetDriverNameWithFeatureTags returns driver name with feature tags -// For example) -// - [Driver: nfs] -// - [Driver: rbd][Feature:Volumes] -func GetDriverNameWithFeatureTags(driver TestDriver) string { - dInfo := driver.GetDriverInfo() - - return fmt.Sprintf("[Driver: %s]%s", dInfo.Name, dInfo.FeatureTag) -} - -// CreateVolume creates volume for test unless dynamicPV test -func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface{} { - switch volType { - case testpatterns.InlineVolume: - fallthrough - case testpatterns.PreprovisionedPV: - if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok { - return pDriver.CreateVolume(volType) - } - case testpatterns.DynamicPV: - // No need to create volume - default: - framework.Failf("Invalid volType specified: %v", volType) - } - return nil -} - -// DeleteVolume deletes volume for test unless dynamicPV test -func DeleteVolume(driver TestDriver, volType testpatterns.TestVolType, testResource interface{}) { - switch volType { - case testpatterns.InlineVolume: - fallthrough - case testpatterns.PreprovisionedPV: - if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok { - pDriver.DeleteVolume(volType, testResource) - } - case testpatterns.DynamicPV: - // No need to delete volume - default: - framework.Failf("Invalid volType specified: %v", volType) - } -} - -// SetCommonDriverParameters sets a common driver parameters to TestDriver -// This function is intended to be called in BeforeEach() inside test loop. -func SetCommonDriverParameters( - driver TestDriver, - f *framework.Framework, - config framework.VolumeTestConfig, -) { - dInfo := driver.GetDriverInfo() - - dInfo.Framework = f - dInfo.Config = config -} - -func getStorageClass( - provisioner string, - parameters map[string]string, - bindingMode *storagev1.VolumeBindingMode, - ns string, - suffix string, -) *storagev1.StorageClass { - if bindingMode == nil { - defaultBindingMode := storagev1.VolumeBindingImmediate - bindingMode = &defaultBindingMode - } - return &storagev1.StorageClass{ - TypeMeta: metav1.TypeMeta{ - Kind: "StorageClass", - }, - ObjectMeta: metav1.ObjectMeta{ - // Name must be unique, so let's base it on namespace name - Name: ns + "-" + suffix, - }, - Provisioner: provisioner, - Parameters: parameters, - VolumeBindingMode: bindingMode, - } -} - -// GetUniqueDriverName returns unique driver name that can be used parallelly in tests -func GetUniqueDriverName(driver TestDriver) string { - return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Framework.UniqueName) -} diff --git a/test/e2e/storage/testsuites/volume_io.go b/test/e2e/storage/testsuites/volume_io.go index 3b291517778..eed39c78c97 100644 --- a/test/e2e/storage/testsuites/volume_io.go +++ b/test/e2e/storage/testsuites/volume_io.go @@ -35,7 +35,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -75,7 +74,7 @@ func (t *volumeIOTestSuite) getTestSuiteInfo() TestSuiteInfo { return t.tsInfo } -func (t *volumeIOTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver drivers.TestDriver) { +func (t *volumeIOTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { } func createVolumeIOTestInput(pattern testpatterns.TestPattern, resource genericVolumeTestResource) volumeIOTestInput { @@ -90,7 +89,7 @@ func createVolumeIOTestInput(pattern testpatterns.TestPattern, resource genericV framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name) } - if dInfo.Capabilities[drivers.CapFsGroup] { + if dInfo.Capabilities[CapFsGroup] { fsGroupVal := int64(1234) fsGroup = &fsGroupVal } @@ -108,7 +107,7 @@ func createVolumeIOTestInput(pattern testpatterns.TestPattern, resource genericV } } -func (t *volumeIOTestSuite) execTest(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (t *volumeIOTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { Context(getTestNameStr(t, pattern), func() { var ( resource genericVolumeTestResource diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index b2f113cd037..86888bd0339 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -27,7 +27,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -62,7 +61,7 @@ func (t *volumeModeTestSuite) getTestSuiteInfo() TestSuiteInfo { return t.tsInfo } -func (t *volumeModeTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver drivers.TestDriver) { +func (t *volumeModeTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { } func createVolumeModeTestInput(pattern testpatterns.TestPattern, resource volumeModeTestResource) volumeModeTestInput { @@ -78,13 +77,13 @@ func createVolumeModeTestInput(pattern testpatterns.TestPattern, resource volume testVolType: pattern.VolType, nodeName: dInfo.Config.ClientNodeName, volMode: pattern.VolMode, - isBlockSupported: dInfo.Capabilities[drivers.CapBlock], + isBlockSupported: dInfo.Capabilities[CapBlock], } } -func getVolumeModeTestFunc(pattern testpatterns.TestPattern, driver drivers.TestDriver) func(*volumeModeTestInput) { +func getVolumeModeTestFunc(pattern testpatterns.TestPattern, driver TestDriver) func(*volumeModeTestInput) { dInfo := driver.GetDriverInfo() - isBlockSupported := dInfo.Capabilities[drivers.CapBlock] + isBlockSupported := dInfo.Capabilities[CapBlock] volMode := pattern.VolMode volType := pattern.VolType @@ -105,7 +104,7 @@ func getVolumeModeTestFunc(pattern testpatterns.TestPattern, driver drivers.Test return nil } -func (t *volumeModeTestSuite) execTest(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (t *volumeModeTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { Context(getTestNameStr(t, pattern), func() { var ( resource volumeModeTestResource @@ -141,7 +140,7 @@ func (t *volumeModeTestSuite) execTest(driver drivers.TestDriver, pattern testpa } type volumeModeTestResource struct { - driver drivers.TestDriver + driver TestDriver sc *storagev1.StorageClass pvc *v1.PersistentVolumeClaim @@ -152,7 +151,7 @@ type volumeModeTestResource struct { var _ TestResource = &volumeModeTestResource{} -func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (s *volumeModeTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { s.driver = driver dInfo := driver.GetDriverInfo() f := dInfo.Framework @@ -168,7 +167,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter ) // Create volume for pre-provisioned volume tests - s.driverTestResource = drivers.CreateVolume(driver, volType) + s.driverTestResource = CreateVolume(driver, volType) switch volType { case testpatterns.PreprovisionedPV: @@ -177,7 +176,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter } else if volMode == v1.PersistentVolumeFilesystem { scName = fmt.Sprintf("%s-%s-sc-for-file", ns.Name, dInfo.Name) } - if pDriver, ok := driver.(drivers.PreprovisionedPVTestDriver); ok { + if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok { pvSource = pDriver.GetPersistentVolumeSource(false, fsType, s.driverTestResource) if pvSource == nil { framework.Skipf("Driver %q does not define PersistentVolumeSource - skipping", dInfo.Name) @@ -189,7 +188,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter s.pvc = framework.MakePersistentVolumeClaim(pvcConfig, ns.Name) } case testpatterns.DynamicPV: - if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { + if dDriver, ok := driver.(DynamicPVTestDriver); ok { s.sc = dDriver.GetDynamicProvisionStorageClass(fsType) if s.sc == nil { framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name) @@ -206,7 +205,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter } } -func (s *volumeModeTestResource) cleanupResource(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (s *volumeModeTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() f := dInfo.Framework cs := f.ClientSet @@ -224,7 +223,7 @@ func (s *volumeModeTestResource) cleanupResource(driver drivers.TestDriver, patt } // Cleanup volume for pre-provisioned volume tests - drivers.DeleteVolume(driver, volType, s.driverTestResource) + DeleteVolume(driver, volType, s.driverTestResource) } type volumeModeTestInput struct { diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index e8bd845bcfd..8c6ef94a546 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -31,7 +31,6 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/testpatterns" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -73,19 +72,19 @@ func (t *volumesTestSuite) getTestSuiteInfo() TestSuiteInfo { return t.tsInfo } -func (t *volumesTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver drivers.TestDriver) { +func (t *volumesTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, driver TestDriver) { } -func skipPersistenceTest(driver drivers.TestDriver) { +func skipPersistenceTest(driver TestDriver) { dInfo := driver.GetDriverInfo() - if !dInfo.Capabilities[drivers.CapPersistence] { + if !dInfo.Capabilities[CapPersistence] { framework.Skipf("Driver %q does not provide persistency - skipping", dInfo.Name) } } -func skipExecTest(driver drivers.TestDriver) { +func skipExecTest(driver TestDriver) { dInfo := driver.GetDriverInfo() - if !dInfo.Capabilities[drivers.CapExec] { + if !dInfo.Capabilities[CapExec] { framework.Skipf("Driver %q does not support exec - skipping", dInfo.Name) } } @@ -101,7 +100,7 @@ func createVolumesTestInput(pattern testpatterns.TestPattern, resource genericVo framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name) } - if dInfo.Capabilities[drivers.CapFsGroup] { + if dInfo.Capabilities[CapFsGroup] { fsGroupVal := int64(1234) fsGroup = &fsGroupVal } @@ -124,7 +123,7 @@ func createVolumesTestInput(pattern testpatterns.TestPattern, resource genericVo } } -func (t *volumesTestSuite) execTest(driver drivers.TestDriver, pattern testpatterns.TestPattern) { +func (t *volumesTestSuite) execTest(driver TestDriver, pattern testpatterns.TestPattern) { Context(getTestNameStr(t, pattern), func() { var ( resource genericVolumeTestResource From 3253e9ab6b9d8a3ea7092dd66c4ac9efaf3a3a13 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 9 Nov 2018 15:46:46 +0100 Subject: [PATCH 4/5] e2e/storage: refactor test config Exposing framework.VolumeTestConfig as part of the testsuite package API was confusing because it was unclear which of the values in it really have an effect. How it was set also was a bit awkward: a test driver had a copy that had to be overwritten at test runtime and then might have been updated and/or overwritten again by the driver. Now testsuites has its own test config structure. It contains the values that might have to be set dynamically at runtime. Instead of overwriting a copy of that struct inside the test driver, the test driver takes some common defaults (specifically, the framework pointer and the prefix) when it gets initialized and then manages its own copy. For example, the hostpath driver has to lock the pods to a single node. framework.VolumeTestConfig is still used internally and test drivers can decide to run tests with a fully populated instance if needed (for example, after setting up an NFS server). --- test/e2e/storage/csi_volumes.go | 25 ++-- test/e2e/storage/drivers/csi.go | 49 ++++--- test/e2e/storage/drivers/csi_objects.go | 6 +- test/e2e/storage/drivers/in_tree.go | 125 ++++++++++++------ test/e2e/storage/in_tree_volumes.go | 23 ++-- test/e2e/storage/testsuites/base.go | 27 +++- .../storage/testsuites/driveroperations.go | 15 +-- test/e2e/storage/testsuites/provisioning.go | 4 +- test/e2e/storage/testsuites/subpath.go | 10 +- test/e2e/storage/testsuites/testdriver.go | 36 ++++- test/e2e/storage/testsuites/volume_io.go | 8 +- test/e2e/storage/testsuites/volumemode.go | 6 +- test/e2e/storage/testsuites/volumes.go | 15 ++- 13 files changed, 211 insertions(+), 138 deletions(-) diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 47f98c60618..71878ead83d 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -43,7 +43,7 @@ import ( ) // List of testDrivers to be executed in below loop -var csiTestDrivers = []func() testsuites.TestDriver{ +var csiTestDrivers = []func(config testsuites.TestConfig) testsuites.TestDriver{ drivers.InitHostPathCSIDriver, drivers.InitGcePDCSIDriver, drivers.InitGcePDExternalCSIDriver, @@ -81,7 +81,11 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { cancel context.CancelFunc cs clientset.Interface ns *v1.Namespace - config framework.VolumeTestConfig + // Common configuration options for each driver. + config = testsuites.TestConfig{ + Framework: f, + Prefix: "csi", + } ) BeforeEach(func() { @@ -89,10 +93,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { cancel = c cs = f.ClientSet ns = f.Namespace - config = framework.VolumeTestConfig{ - Namespace: ns.Name, - Prefix: "csi", - } + // Debugging of the following tests heavily depends on the log output // of the different containers. Therefore include all of that in log // files (when using --report-dir, as in the CI) or the output stream @@ -125,13 +126,12 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { }) for _, initDriver := range csiTestDrivers { - curDriver := initDriver() + curDriver := initDriver(config) Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() { driver := curDriver BeforeEach(func() { // setupDriver - testsuites.SetCommonDriverParameters(driver, f, config) driver.CreateDriver() }) @@ -140,7 +140,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { driver.CleanupDriver() }) - testsuites.RunTestSuite(f, config, driver, csiTestSuites, csiTunePattern) + testsuites.RunTestSuite(f, driver, csiTestSuites, csiTunePattern) }) } @@ -155,8 +155,11 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { BeforeEach(func() { cs = f.ClientSet csics = f.CSIClientSet - driver = drivers.InitHostPathCSIDriver() - testsuites.SetCommonDriverParameters(driver, f, config) + config := testsuites.TestConfig{ + Framework: f, + Prefix: "csi-attach", + } + driver = drivers.InitHostPathCSIDriver(config) driver.CreateDriver() }) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 62187a0f6c8..357ef2318ce 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -55,7 +55,7 @@ type hostpathCSIDriver struct { manifests []string } -func initHostPathCSIDriver(name string, manifests ...string) testsuites.TestDriver { +func initHostPathCSIDriver(name string, config testsuites.TestConfig, manifests ...string) testsuites.TestDriver { return &hostpathCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: name, @@ -67,6 +67,8 @@ func initHostPathCSIDriver(name string, manifests ...string) testsuites.TestDriv Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, }, + + Config: config, }, manifests: manifests, } @@ -76,8 +78,8 @@ var _ testsuites.TestDriver = &hostpathCSIDriver{} var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{} // InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface -func InitHostPathCSIDriver() testsuites.TestDriver { - return initHostPathCSIDriver("csi-hostpath", +func InitHostPathCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { + return initHostPathCSIDriver("csi-hostpath", config, "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", @@ -98,7 +100,7 @@ func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { provisioner := testsuites.GetUniqueDriverName(h) parameters := map[string]string{} - ns := h.driverInfo.Framework.Namespace.Name + ns := h.driverInfo.Config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", provisioner) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) @@ -110,14 +112,13 @@ func (h *hostpathCSIDriver) GetClaimSize() string { func (h *hostpathCSIDriver) CreateDriver() { By(fmt.Sprintf("deploying %s driver", h.driverInfo.Name)) - f := h.driverInfo.Framework + f := h.driverInfo.Config.Framework cs := f.ClientSet - // pods should be scheduled on the node + // The hostpath CSI driver only works when everything runs on the same node. nodes := framework.GetReadySchedulableNodesOrDie(cs) - node := nodes.Items[rand.Intn(len(nodes.Items))] - h.driverInfo.Config.ClientNodeName = node.Name - h.driverInfo.Config.ServerNodeName = node.Name + nodeName := nodes.Items[rand.Intn(len(nodes.Items))].Name + h.driverInfo.Config.ClientNodeName = nodeName // TODO (?): the storage.csi.image.version and storage.csi.image.registry // settings are ignored for this test. We could patch the image definitions. @@ -126,10 +127,10 @@ func (h *hostpathCSIDriver) CreateDriver() { NewDriverName: testsuites.GetUniqueDriverName(h), DriverContainerName: "hostpath", ProvisionerContainerName: "csi-provisioner", - NodeName: h.driverInfo.Config.ServerNodeName, + NodeName: nodeName, } - cleanup, err := h.driverInfo.Framework.CreateFromManifests(func(item interface{}) error { - return utils.PatchCSIDeployment(h.driverInfo.Framework, o, item) + cleanup, err := h.driverInfo.Config.Framework.CreateFromManifests(func(item interface{}) error { + return utils.PatchCSIDeployment(h.driverInfo.Config.Framework, o, item) }, h.manifests...) h.cleanup = cleanup @@ -146,8 +147,8 @@ func (h *hostpathCSIDriver) CleanupDriver() { } // InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests. -func InitHostPathV0CSIDriver() testsuites.TestDriver { - return initHostPathCSIDriver("csi-hostpath-v0", +func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver { + return initHostPathCSIDriver("csi-hostpath-v0", config, "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", @@ -168,7 +169,7 @@ var _ testsuites.TestDriver = &gcePDCSIDriver{} var _ testsuites.DynamicPVTestDriver = &gcePDCSIDriver{} // InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface -func InitGcePDCSIDriver() testsuites.TestDriver { +func InitGcePDCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { return &gcePDCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: "pd.csi.storage.gke.io", @@ -186,6 +187,8 @@ func InitGcePDCSIDriver() testsuites.TestDriver { testsuites.CapFsGroup: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -195,13 +198,13 @@ func (g *gcePDCSIDriver) GetDriverInfo() *testsuites.DriverInfo { } func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { - f := g.driverInfo.Framework + f := g.driverInfo.Config.Framework framework.SkipUnlessProviderIs("gce", "gke") framework.SkipIfMultizone(f.ClientSet) } func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { - ns := g.driverInfo.Framework.Namespace.Name + ns := g.driverInfo.Config.Framework.Namespace.Name provisioner := g.driverInfo.Name suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) @@ -228,9 +231,9 @@ func (g *gcePDCSIDriver) CreateDriver() { // DriverContainerName: "gce-driver", // ProvisionerContainerName: "csi-external-provisioner", // } - createGCESecrets(g.driverInfo.Framework.ClientSet, g.driverInfo.Config) + createGCESecrets(g.driverInfo.Config.Framework.ClientSet, g.driverInfo.Config.Framework.Namespace.Name) - cleanup, err := g.driverInfo.Framework.CreateFromManifests(nil, + cleanup, err := g.driverInfo.Config.Framework.CreateFromManifests(nil, "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", @@ -260,7 +263,7 @@ var _ testsuites.TestDriver = &gcePDExternalCSIDriver{} var _ testsuites.DynamicPVTestDriver = &gcePDExternalCSIDriver{} // InitGcePDExternalCSIDriver returns gcePDExternalCSIDriver that implements TestDriver interface -func InitGcePDExternalCSIDriver() testsuites.TestDriver { +func InitGcePDExternalCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { return &gcePDExternalCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: "pd.csi.storage.gke.io", @@ -280,6 +283,8 @@ func InitGcePDExternalCSIDriver() testsuites.TestDriver { testsuites.CapFsGroup: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -290,11 +295,11 @@ func (g *gcePDExternalCSIDriver) GetDriverInfo() *testsuites.DriverInfo { func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { framework.SkipUnlessProviderIs("gce", "gke") - framework.SkipIfMultizone(g.driverInfo.Framework.ClientSet) + framework.SkipIfMultizone(g.driverInfo.Config.Framework.ClientSet) } func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { - ns := g.driverInfo.Framework.Namespace.Name + ns := g.driverInfo.Config.Framework.Namespace.Name provisioner := g.driverInfo.Name suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) diff --git a/test/e2e/storage/drivers/csi_objects.go b/test/e2e/storage/drivers/csi_objects.go index 1a4c2af592f..9960da8add4 100644 --- a/test/e2e/storage/drivers/csi_objects.go +++ b/test/e2e/storage/drivers/csi_objects.go @@ -80,7 +80,7 @@ func shredFile(filePath string) { // createGCESecrets downloads the GCP IAM Key for the default compute service account // and puts it in a secret for the GCE PD CSI Driver to consume -func createGCESecrets(client clientset.Interface, config framework.VolumeTestConfig) { +func createGCESecrets(client clientset.Interface, ns string) { saEnv := "E2E_GOOGLE_APPLICATION_CREDENTIALS" saFile := fmt.Sprintf("/tmp/%s/cloud-sa.json", string(uuid.NewUUID())) @@ -107,7 +107,7 @@ func createGCESecrets(client clientset.Interface, config framework.VolumeTestCon s := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "cloud-sa", - Namespace: config.Namespace, + Namespace: ns, }, Type: v1.SecretTypeOpaque, Data: map[string][]byte{ @@ -115,7 +115,7 @@ func createGCESecrets(client clientset.Interface, config framework.VolumeTestCon }, } - _, err = client.CoreV1().Secrets(config.Namespace).Create(s) + _, err = client.CoreV1().Secrets(ns).Create(s) if !apierrors.IsAlreadyExists(err) { framework.ExpectNoError(err, "Failed to create Secret %v", s.GetName()) } diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index d121f3e4fa9..20d192502e2 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -81,7 +81,7 @@ var _ testsuites.PreprovisionedPVTestDriver = &nfsDriver{} var _ testsuites.DynamicPVTestDriver = &nfsDriver{} // InitNFSDriver returns nfsDriver that implements TestDriver interface -func InitNFSDriver() testsuites.TestDriver { +func InitNFSDriver(config testsuites.TestConfig) testsuites.TestDriver { return &nfsDriver{ driverInfo: testsuites.DriverInfo{ Name: "nfs", @@ -95,6 +95,8 @@ func InitNFSDriver() testsuites.TestDriver { testsuites.CapPersistence: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -133,7 +135,7 @@ func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, test func (n *nfsDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { provisioner := n.externalPluginName parameters := map[string]string{"mountOptions": "vers=4.1"} - ns := n.driverInfo.Framework.Namespace.Name + ns := n.driverInfo.Config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", n.driverInfo.Name) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) @@ -144,11 +146,14 @@ func (n *nfsDriver) GetClaimSize() string { } func (n *nfsDriver) CreateDriver() { - f := n.driverInfo.Framework + f := n.driverInfo.Config.Framework cs := f.ClientSet ns := f.Namespace n.externalPluginName = fmt.Sprintf("example.com/nfs-%s", ns.Name) + // Reset config. It might have been modified by a previous CreateVolume call. + n.driverInfo.Config.ServerConfig = nil + // TODO(mkimuram): cluster-admin gives too much right but system:persistent-volume-provisioner // is not enough. We should create new clusterrole for testing. framework.BindClusterRole(cs.RbacV1beta1(), "cluster-admin", ns.Name, @@ -164,7 +169,7 @@ func (n *nfsDriver) CreateDriver() { } func (n *nfsDriver) CleanupDriver() { - f := n.driverInfo.Framework + f := n.driverInfo.Config.Framework cs := f.ClientSet ns := f.Namespace @@ -174,7 +179,7 @@ func (n *nfsDriver) CleanupDriver() { } func (n *nfsDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := n.driverInfo.Framework + f := n.driverInfo.Config.Framework cs := f.ClientSet ns := f.Namespace @@ -186,7 +191,7 @@ func (n *nfsDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { fallthrough case testpatterns.PreprovisionedPV: config, serverPod, serverIP := framework.NewNFSServer(cs, ns.Name, []string{}) - n.driverInfo.Config = config + n.driverInfo.Config.ServerConfig = &config return &nfsTestResource{ serverIP: serverIP, serverPod: serverPod, @@ -200,7 +205,7 @@ func (n *nfsDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { } func (n *nfsDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - f := n.driverInfo.Framework + f := n.driverInfo.Config.Framework ntr, ok := testResource.(*nfsTestResource) Expect(ok).To(BeTrue(), "Failed to cast test resource to NFS Test Resource") @@ -233,7 +238,7 @@ var _ testsuites.InlineVolumeTestDriver = &glusterFSDriver{} var _ testsuites.PreprovisionedPVTestDriver = &glusterFSDriver{} // InitGlusterFSDriver returns glusterFSDriver that implements TestDriver interface -func InitGlusterFSDriver() testsuites.TestDriver { +func InitGlusterFSDriver(config testsuites.TestConfig) testsuites.TestDriver { return &glusterFSDriver{ driverInfo: testsuites.DriverInfo{ Name: "gluster", @@ -245,6 +250,8 @@ func InitGlusterFSDriver() testsuites.TestDriver { testsuites.CapPersistence: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -291,18 +298,20 @@ func (g *glusterFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string } func (g *glusterFSDriver) CreateDriver() { + // Reset config. It might have been modified by a previous CreateVolume call. + g.driverInfo.Config.ServerConfig = nil } func (g *glusterFSDriver) CleanupDriver() { } func (g *glusterFSDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := g.driverInfo.Framework + f := g.driverInfo.Config.Framework cs := f.ClientSet ns := f.Namespace config, serverPod, _ := framework.NewGlusterfsServer(cs, ns.Name) - g.driverInfo.Config = config + g.driverInfo.Config.ServerConfig = &config return &glusterTestResource{ prefix: config.Prefix, serverPod: serverPod, @@ -310,7 +319,7 @@ func (g *glusterFSDriver) CreateVolume(volType testpatterns.TestVolType) interfa } func (g *glusterFSDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - f := g.driverInfo.Framework + f := g.driverInfo.Config.Framework cs := f.ClientSet ns := f.Namespace @@ -350,7 +359,7 @@ var _ testsuites.InlineVolumeTestDriver = &iSCSIDriver{} var _ testsuites.PreprovisionedPVTestDriver = &iSCSIDriver{} // InitISCSIDriver returns iSCSIDriver that implements TestDriver interface -func InitISCSIDriver() testsuites.TestDriver { +func InitISCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { return &iSCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: "iscsi", @@ -369,6 +378,8 @@ func InitISCSIDriver() testsuites.TestDriver { testsuites.CapBlock: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -418,18 +429,20 @@ func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, te } func (i *iSCSIDriver) CreateDriver() { + // Reset config. It might have been modified by a previous CreateVolume call. + i.driverInfo.Config.ServerConfig = nil } func (i *iSCSIDriver) CleanupDriver() { } func (i *iSCSIDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := i.driverInfo.Framework + f := i.driverInfo.Config.Framework cs := f.ClientSet ns := f.Namespace config, serverPod, serverIP := framework.NewISCSIServer(cs, ns.Name) - i.driverInfo.Config = config + i.driverInfo.Config.ServerConfig = &config return &iSCSITestResource{ serverPod: serverPod, serverIP: serverIP, @@ -437,7 +450,7 @@ func (i *iSCSIDriver) CreateVolume(volType testpatterns.TestVolType) interface{} } func (i *iSCSIDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - f := i.driverInfo.Framework + f := i.driverInfo.Config.Framework itr, ok := testResource.(*iSCSITestResource) Expect(ok).To(BeTrue(), "Failed to cast test resource to iSCSI Test Resource") @@ -462,7 +475,7 @@ var _ testsuites.InlineVolumeTestDriver = &rbdDriver{} var _ testsuites.PreprovisionedPVTestDriver = &rbdDriver{} // InitRbdDriver returns rbdDriver that implements TestDriver interface -func InitRbdDriver() testsuites.TestDriver { +func InitRbdDriver(config testsuites.TestConfig) testsuites.TestDriver { return &rbdDriver{ driverInfo: testsuites.DriverInfo{ Name: "rbd", @@ -481,6 +494,8 @@ func InitRbdDriver() testsuites.TestDriver { testsuites.CapBlock: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -515,7 +530,7 @@ func (r *rbdDriver) GetVolumeSource(readOnly bool, fsType string, testResource i } func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { - f := r.driverInfo.Framework + f := r.driverInfo.Config.Framework ns := f.Namespace rtr, ok := testResource.(*rbdTestResource) @@ -541,18 +556,20 @@ func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, test } func (r *rbdDriver) CreateDriver() { + // Reset config. It might have been modified by a previous CreateVolume call. + r.driverInfo.Config.ServerConfig = nil } func (r *rbdDriver) CleanupDriver() { } func (r *rbdDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := r.driverInfo.Framework + f := r.driverInfo.Config.Framework cs := f.ClientSet ns := f.Namespace config, serverPod, secret, serverIP := framework.NewRBDServer(cs, ns.Name) - r.driverInfo.Config = config + r.driverInfo.Config.ServerConfig = &config return &rbdTestResource{ serverPod: serverPod, serverIP: serverIP, @@ -561,7 +578,7 @@ func (r *rbdDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { } func (r *rbdDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - f := r.driverInfo.Framework + f := r.driverInfo.Config.Framework rtr, ok := testResource.(*rbdTestResource) Expect(ok).To(BeTrue(), "Failed to cast test resource to RBD Test Resource") @@ -590,7 +607,7 @@ var _ testsuites.InlineVolumeTestDriver = &cephFSDriver{} var _ testsuites.PreprovisionedPVTestDriver = &cephFSDriver{} // InitCephFSDriver returns cephFSDriver that implements TestDriver interface -func InitCephFSDriver() testsuites.TestDriver { +func InitCephFSDriver(config testsuites.TestConfig) testsuites.TestDriver { return &cephFSDriver{ driverInfo: testsuites.DriverInfo{ Name: "ceph", @@ -603,6 +620,8 @@ func InitCephFSDriver() testsuites.TestDriver { testsuites.CapPersistence: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -631,7 +650,7 @@ func (c *cephFSDriver) GetVolumeSource(readOnly bool, fsType string, testResourc } func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { - f := c.driverInfo.Framework + f := c.driverInfo.Config.Framework ns := f.Namespace ctr, ok := testResource.(*cephTestResource) @@ -651,18 +670,20 @@ func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, t } func (c *cephFSDriver) CreateDriver() { + // Reset config. It might have been modified by a previous CreateVolume call. + c.driverInfo.Config.ServerConfig = nil } func (c *cephFSDriver) CleanupDriver() { } func (c *cephFSDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := c.driverInfo.Framework + f := c.driverInfo.Config.Framework cs := f.ClientSet ns := f.Namespace config, serverPod, secret, serverIP := framework.NewRBDServer(cs, ns.Name) - c.driverInfo.Config = config + c.driverInfo.Config.ServerConfig = &config return &cephTestResource{ serverPod: serverPod, serverIP: serverIP, @@ -671,7 +692,7 @@ func (c *cephFSDriver) CreateVolume(volType testpatterns.TestVolType) interface{ } func (c *cephFSDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - f := c.driverInfo.Framework + f := c.driverInfo.Config.Framework ctr, ok := testResource.(*cephTestResource) Expect(ok).To(BeTrue(), "Failed to cast test resource to Ceph Test Resource") @@ -691,7 +712,7 @@ var _ testsuites.PreprovisionedVolumeTestDriver = &hostPathDriver{} var _ testsuites.InlineVolumeTestDriver = &hostPathDriver{} // InitHostPathDriver returns hostPathDriver that implements TestDriver interface -func InitHostPathDriver() testsuites.TestDriver { +func InitHostPathDriver(config testsuites.TestConfig) testsuites.TestDriver { return &hostPathDriver{ driverInfo: testsuites.DriverInfo{ Name: "hostPath", @@ -702,6 +723,8 @@ func InitHostPathDriver() testsuites.TestDriver { Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, }, + + Config: config, }, } } @@ -732,7 +755,7 @@ func (h *hostPathDriver) CleanupDriver() { } func (h *hostPathDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := h.driverInfo.Framework + f := h.driverInfo.Config.Framework cs := f.ClientSet // pods should be scheduled on the node @@ -763,7 +786,7 @@ var _ testsuites.PreprovisionedVolumeTestDriver = &hostPathSymlinkDriver{} var _ testsuites.InlineVolumeTestDriver = &hostPathSymlinkDriver{} // InitHostPathSymlinkDriver returns hostPathSymlinkDriver that implements TestDriver interface -func InitHostPathSymlinkDriver() testsuites.TestDriver { +func InitHostPathSymlinkDriver(config testsuites.TestConfig) testsuites.TestDriver { return &hostPathSymlinkDriver{ driverInfo: testsuites.DriverInfo{ Name: "hostPathSymlink", @@ -774,6 +797,8 @@ func InitHostPathSymlinkDriver() testsuites.TestDriver { Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, }, + + Config: config, }, } } @@ -807,7 +832,7 @@ func (h *hostPathSymlinkDriver) CleanupDriver() { } func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := h.driverInfo.Framework + f := h.driverInfo.Config.Framework cs := f.ClientSet sourcePath := fmt.Sprintf("/tmp/%v", f.Namespace.Name) @@ -875,7 +900,7 @@ func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) i } func (h *hostPathSymlinkDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - f := h.driverInfo.Framework + f := h.driverInfo.Config.Framework htr, ok := testResource.(*hostPathSymlinkTestResource) Expect(ok).To(BeTrue(), "Failed to cast test resource to Hostpath Symlink Test Resource") @@ -903,7 +928,7 @@ var _ testsuites.PreprovisionedVolumeTestDriver = &emptydirDriver{} var _ testsuites.InlineVolumeTestDriver = &emptydirDriver{} // InitEmptydirDriver returns emptydirDriver that implements TestDriver interface -func InitEmptydirDriver() testsuites.TestDriver { +func InitEmptydirDriver(config testsuites.TestConfig) testsuites.TestDriver { return &emptydirDriver{ driverInfo: testsuites.DriverInfo{ Name: "emptydir", @@ -914,6 +939,8 @@ func InitEmptydirDriver() testsuites.TestDriver { Capabilities: map[testsuites.Capability]bool{ testsuites.CapExec: true, }, + + Config: config, }, } } @@ -969,7 +996,7 @@ var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{} var _ testsuites.DynamicPVTestDriver = &cinderDriver{} // InitCinderDriver returns cinderDriver that implements TestDriver interface -func InitCinderDriver() testsuites.TestDriver { +func InitCinderDriver(config testsuites.TestConfig) testsuites.TestDriver { return &cinderDriver{ driverInfo: testsuites.DriverInfo{ Name: "cinder", @@ -983,6 +1010,8 @@ func InitCinderDriver() testsuites.TestDriver { testsuites.CapFsGroup: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -1033,7 +1062,7 @@ func (c *cinderDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1 if fsType != "" { parameters["fsType"] = fsType } - ns := c.driverInfo.Framework.Namespace.Name + ns := c.driverInfo.Config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", c.driverInfo.Name) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) @@ -1050,7 +1079,7 @@ func (c *cinderDriver) CleanupDriver() { } func (c *cinderDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := c.driverInfo.Framework + f := c.driverInfo.Config.Framework ns := f.Namespace // We assume that namespace.Name is a random string @@ -1129,7 +1158,7 @@ var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{} var _ testsuites.DynamicPVTestDriver = &gcePdDriver{} // InitGceDriver returns gcePdDriver that implements TestDriver interface -func InitGcePdDriver() testsuites.TestDriver { +func InitGcePdDriver(config testsuites.TestConfig) testsuites.TestDriver { return &gcePdDriver{ driverInfo: testsuites.DriverInfo{ Name: "gcepd", @@ -1148,6 +1177,8 @@ func InitGcePdDriver() testsuites.TestDriver { testsuites.CapBlock: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -1199,7 +1230,7 @@ func (g *gcePdDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1. if fsType != "" { parameters["fsType"] = fsType } - ns := g.driverInfo.Framework.Namespace.Name + ns := g.driverInfo.Config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) @@ -1219,7 +1250,7 @@ func (g *gcePdDriver) CreateVolume(volType testpatterns.TestVolType) interface{} if volType == testpatterns.InlineVolume { // PD will be created in framework.TestContext.CloudConfig.Zone zone, // so pods should be also scheduled there. - g.driverInfo.Config.NodeSelector = map[string]string{ + g.driverInfo.Config.ClientNodeSelector = map[string]string{ kubeletapis.LabelZoneFailureDomain: framework.TestContext.CloudConfig.Zone, } } @@ -1254,7 +1285,7 @@ var _ testsuites.PreprovisionedPVTestDriver = &vSphereDriver{} var _ testsuites.DynamicPVTestDriver = &vSphereDriver{} // InitVSphereDriver returns vSphereDriver that implements TestDriver interface -func InitVSphereDriver() testsuites.TestDriver { +func InitVSphereDriver(config testsuites.TestConfig) testsuites.TestDriver { return &vSphereDriver{ driverInfo: testsuites.DriverInfo{ Name: "vSphere", @@ -1268,6 +1299,8 @@ func InitVSphereDriver() testsuites.TestDriver { testsuites.CapFsGroup: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -1325,7 +1358,7 @@ func (v *vSphereDriver) GetDynamicProvisionStorageClass(fsType string) *storagev if fsType != "" { parameters["fsType"] = fsType } - ns := v.driverInfo.Framework.Namespace.Name + ns := v.driverInfo.Config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", v.driverInfo.Name) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) @@ -1342,7 +1375,7 @@ func (v *vSphereDriver) CleanupDriver() { } func (v *vSphereDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { - f := v.driverInfo.Framework + f := v.driverInfo.Config.Framework vspheretest.Bootstrap(f) nodeInfo := vspheretest.GetReadySchedulableRandomNodeInfo() volumePath, err := nodeInfo.VSphere.CreateVolume(&vspheretest.VolumeOptions{}, nodeInfo.DataCenterRef) @@ -1376,7 +1409,7 @@ var _ testsuites.PreprovisionedPVTestDriver = &azureDriver{} var _ testsuites.DynamicPVTestDriver = &azureDriver{} // InitAzureDriver returns azureDriver that implements TestDriver interface -func InitAzureDriver() testsuites.TestDriver { +func InitAzureDriver(config testsuites.TestConfig) testsuites.TestDriver { return &azureDriver{ driverInfo: testsuites.DriverInfo{ Name: "azure", @@ -1391,6 +1424,8 @@ func InitAzureDriver() testsuites.TestDriver { testsuites.CapBlock: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -1447,7 +1482,7 @@ func (a *azureDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1. if fsType != "" { parameters["fsType"] = fsType } - ns := a.driverInfo.Framework.Namespace.Name + ns := a.driverInfo.Config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) @@ -1495,7 +1530,7 @@ var _ testsuites.TestDriver = &awsDriver{} var _ testsuites.DynamicPVTestDriver = &awsDriver{} // InitAwsDriver returns awsDriver that implements TestDriver interface -func InitAwsDriver() testsuites.TestDriver { +func InitAwsDriver(config testsuites.TestConfig) testsuites.TestDriver { return &awsDriver{ driverInfo: testsuites.DriverInfo{ Name: "aws", @@ -1511,6 +1546,8 @@ func InitAwsDriver() testsuites.TestDriver { testsuites.CapBlock: true, testsuites.CapExec: true, }, + + Config: config, }, } } @@ -1558,7 +1595,7 @@ func (a *awsDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.St if fsType != "" { parameters["fsType"] = fsType } - ns := a.driverInfo.Framework.Namespace.Name + ns := a.driverInfo.Config.Framework.Namespace.Name suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name) return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 11c9e73459d..c9f6401578f 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -18,7 +18,6 @@ package storage import ( . "github.com/onsi/ginkgo" - "k8s.io/api/core/v1" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/testpatterns" @@ -27,7 +26,7 @@ import ( ) // List of testDrivers to be executed in below loop -var testDrivers = []func() testsuites.TestDriver{ +var testDrivers = []func(config testsuites.TestConfig) testsuites.TestDriver{ drivers.InitNFSDriver, drivers.InitGlusterFSDriver, drivers.InitISCSIDriver, @@ -61,26 +60,20 @@ var _ = utils.SIGDescribe("In-tree Volumes", func() { f := framework.NewDefaultFramework("volumes") var ( - ns *v1.Namespace - config framework.VolumeTestConfig + // Common configuration options for all drivers. + config = testsuites.TestConfig{ + Framework: f, + Prefix: "in-tree", + } ) - BeforeEach(func() { - ns = f.Namespace - config = framework.VolumeTestConfig{ - Namespace: ns.Name, - Prefix: "volume", - } - }) - for _, initDriver := range testDrivers { - curDriver := initDriver() + curDriver := initDriver(config) Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() { driver := curDriver BeforeEach(func() { // setupDriver - testsuites.SetCommonDriverParameters(driver, f, config) driver.CreateDriver() }) @@ -89,7 +82,7 @@ var _ = utils.SIGDescribe("In-tree Volumes", func() { driver.CleanupDriver() }) - testsuites.RunTestSuite(f, config, driver, testSuites, intreeTunePattern) + testsuites.RunTestSuite(f, driver, testSuites, intreeTunePattern) }) } }) diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index c257b849620..8638a90cfe1 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -65,7 +65,7 @@ func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string { } // RunTestSuite runs all testpatterns of all testSuites for a driver -func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) { +func RunTestSuite(f *framework.Framework, driver TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) { for _, testSuiteInit := range tsInits { suite := testSuiteInit() patterns := tunePatternFunc(suite.getTestSuiteInfo().testPatterns) @@ -136,7 +136,7 @@ var _ TestResource = &genericVolumeTestResource{} func (r *genericVolumeTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { r.driver = driver dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework cs := f.ClientSet fsType := pattern.FsType volType := pattern.VolType @@ -189,7 +189,7 @@ func (r *genericVolumeTestResource) setupResource(driver TestDriver, pattern tes // cleanupResource cleans up genericVolumeTestResource func (r *genericVolumeTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework volType := pattern.VolType if r.pvc != nil || r.pv != nil { @@ -323,3 +323,24 @@ func deleteStorageClass(cs clientset.Interface, className string) { Expect(err).NotTo(HaveOccurred()) } } + +// convertTestConfig returns a framework test config with the +// parameters specified for the testsuite or (if available) the +// dynamically created config for the volume server. +// +// This is done because TestConfig is the public API for +// the testsuites package whereas framework.VolumeTestConfig is merely +// an implementation detail. It contains fields that have no effect, +// which makes it unsuitable for use in the testsuits public API. +func convertTestConfig(in *TestConfig) framework.VolumeTestConfig { + if in.ServerConfig != nil { + return *in.ServerConfig + } + + return framework.VolumeTestConfig{ + Namespace: in.Framework.Namespace.Name, + Prefix: in.Prefix, + ClientNodeName: in.ClientNodeName, + NodeSelector: in.ClientNodeSelector, + } +} diff --git a/test/e2e/storage/testsuites/driveroperations.go b/test/e2e/storage/testsuites/driveroperations.go index 41c2d3eb22f..1e233b48387 100644 --- a/test/e2e/storage/testsuites/driveroperations.go +++ b/test/e2e/storage/testsuites/driveroperations.go @@ -68,19 +68,6 @@ func DeleteVolume(driver TestDriver, volType testpatterns.TestVolType, testResou } } -// SetCommonDriverParameters sets a common driver parameters to TestDriver -// This function is intended to be called in BeforeEach() inside test loop. -func SetCommonDriverParameters( - driver TestDriver, - f *framework.Framework, - config framework.VolumeTestConfig, -) { - dInfo := driver.GetDriverInfo() - - dInfo.Framework = f - dInfo.Config = config -} - // GetStorageClass constructs a new StorageClass instance // with a unique name that is based on namespace + suffix. func GetStorageClass( @@ -110,5 +97,5 @@ func GetStorageClass( // GetUniqueDriverName returns unique driver name that can be used parallelly in tests func GetUniqueDriverName(driver TestDriver) string { - return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Framework.UniqueName) + return fmt.Sprintf("%s-%s", driver.GetDriverInfo().Name, driver.GetDriverInfo().Config.Framework.UniqueName) } diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 7524af9fc1c..527218d23ca 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -86,7 +86,7 @@ func createProvisioningTestInput(driver TestDriver, pattern testpatterns.TestPat ClaimSize: resource.claimSize, ExpectedSize: resource.claimSize, }, - cs: driver.GetDriverInfo().Framework.ClientSet, + cs: driver.GetDriverInfo().Config.Framework.ClientSet, pvc: resource.pvc, sc: resource.sc, dInfo: driver.GetDriverInfo(), @@ -151,7 +151,7 @@ func (p *provisioningTestResource) setupResource(driver TestDriver, pattern test } p.driver = driver p.claimSize = dDriver.GetClaimSize() - p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name) + p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Config.Framework.Namespace.Name) p.pvc.Spec.StorageClassName = &p.sc.Name framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc) } diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index 50daffeaeee..b8e957884a1 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -77,7 +77,7 @@ func (s *subPathTestSuite) skipUnsupportedTest(pattern testpatterns.TestPattern, func createSubPathTestInput(pattern testpatterns.TestPattern, resource subPathTestResource) subPathTestInput { driver := resource.driver dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework subPath := f.Namespace.Name subPathDir := filepath.Join(volumePath, subPath) @@ -139,7 +139,7 @@ var _ TestResource = &subPathTestResource{} func (s *subPathTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { s.driver = driver dInfo := s.driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework fsType := pattern.FsType volType := pattern.VolType @@ -174,16 +174,16 @@ func (s *subPathTestResource) setupResource(driver TestDriver, pattern testpatte config := dInfo.Config s.pod = SubpathTestPod(f, subPath, s.volType, s.volSource, true) s.pod.Spec.NodeName = config.ClientNodeName - s.pod.Spec.NodeSelector = config.NodeSelector + s.pod.Spec.NodeSelector = config.ClientNodeSelector s.formatPod = volumeFormatPod(f, s.volSource) s.formatPod.Spec.NodeName = config.ClientNodeName - s.formatPod.Spec.NodeSelector = config.NodeSelector + s.formatPod.Spec.NodeSelector = config.ClientNodeSelector } func (s *subPathTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework // Cleanup subPath test dependent resource By("Deleting pod") diff --git a/test/e2e/storage/testsuites/testdriver.go b/test/e2e/storage/testsuites/testdriver.go index 6af6ba70855..beb8ea1a429 100644 --- a/test/e2e/storage/testsuites/testdriver.go +++ b/test/e2e/storage/testsuites/testdriver.go @@ -99,9 +99,35 @@ type DriverInfo struct { RequiredMountOption sets.String // Map of string for required mount option (Optional) Capabilities map[Capability]bool // Map that represents plugin capabilities - // Parameters below will be set inside test loop by using SetCommonDriverParameters. - // Drivers that implement TestDriver is required to set all the above parameters - // and return DriverInfo on GetDriverInfo() call. - Framework *framework.Framework // Framework for the test - Config framework.VolumeTestConfig // VolumeTestConfig for thet test + Config TestConfig // Test configuration for the current test. +} + +// TestConfig represents parameters that control test execution. +// They can still be modified after defining tests, for example +// in a BeforeEach or when creating the driver. +type TestConfig struct { + // Some short word that gets inserted into dynamically + // generated entities (pods, paths) as first part of the name + // to make debugging easier. Can be the same for different + // tests inside the test suite. + Prefix string + + // The framework instance allocated for the current test. + Framework *framework.Framework + + // If non-empty, then pods using a volume will be scheduled + // onto the node with this name. Otherwise Kubernetes will + // pick a node. + ClientNodeName string + + // Some tests also support scheduling pods onto nodes with + // these label/value pairs. As not all tests use this field, + // a driver that absolutely needs the pods on a specific + // node must use ClientNodeName. + ClientNodeSelector map[string]string + + // Some test drivers initialize a storage server. This is + // the configuration that then has to be used to run tests. + // The values above are ignored for such tests. + ServerConfig *framework.VolumeTestConfig } diff --git a/test/e2e/storage/testsuites/volume_io.go b/test/e2e/storage/testsuites/volume_io.go index eed39c78c97..ad32f84fa5f 100644 --- a/test/e2e/storage/testsuites/volume_io.go +++ b/test/e2e/storage/testsuites/volume_io.go @@ -81,7 +81,7 @@ func createVolumeIOTestInput(pattern testpatterns.TestPattern, resource genericV var fsGroup *int64 driver := resource.driver dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework fileSizes := createFileSizes(dInfo.MaxFileSize) volSource := resource.volSource @@ -97,7 +97,7 @@ func createVolumeIOTestInput(pattern testpatterns.TestPattern, resource genericV return volumeIOTestInput{ f: f, name: dInfo.Name, - config: dInfo.Config, + config: &dInfo.Config, volSource: *volSource, testFile: fmt.Sprintf("%s_io_test_%s", dInfo.Name, f.Namespace.Name), podSec: v1.PodSecurityContext{ @@ -142,7 +142,7 @@ func (t *volumeIOTestSuite) execTest(driver TestDriver, pattern testpatterns.Tes type volumeIOTestInput struct { f *framework.Framework name string - config framework.VolumeTestConfig + config *TestConfig volSource v1.VolumeSource testFile string podSec v1.PodSecurityContext @@ -154,7 +154,7 @@ func execTestVolumeIO(input *volumeIOTestInput) { f := input.f cs := f.ClientSet - err := testVolumeIO(f, cs, input.config, input.volSource, &input.podSec, input.testFile, input.fileSizes) + err := testVolumeIO(f, cs, convertTestConfig(input.config), input.volSource, &input.podSec, input.testFile, input.fileSizes) Expect(err).NotTo(HaveOccurred()) }) } diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index 86888bd0339..03eb2a43b09 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -67,7 +67,7 @@ func (t *volumeModeTestSuite) skipUnsupportedTest(pattern testpatterns.TestPatte func createVolumeModeTestInput(pattern testpatterns.TestPattern, resource volumeModeTestResource) volumeModeTestInput { driver := resource.driver dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework return volumeModeTestInput{ f: f, @@ -154,7 +154,7 @@ var _ TestResource = &volumeModeTestResource{} func (s *volumeModeTestResource) setupResource(driver TestDriver, pattern testpatterns.TestPattern) { s.driver = driver dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework ns := f.Namespace fsType := pattern.FsType volBindMode := storagev1.VolumeBindingImmediate @@ -207,7 +207,7 @@ func (s *volumeModeTestResource) setupResource(driver TestDriver, pattern testpa func (s *volumeModeTestResource) cleanupResource(driver TestDriver, pattern testpatterns.TestPattern) { dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework cs := f.ClientSet ns := f.Namespace volType := pattern.VolType diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index 8c6ef94a546..1d890e085a9 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -93,7 +93,7 @@ func createVolumesTestInput(pattern testpatterns.TestPattern, resource genericVo var fsGroup *int64 driver := resource.driver dInfo := driver.GetDriverInfo() - f := dInfo.Framework + f := dInfo.Config.Framework volSource := resource.volSource if volSource == nil { @@ -108,7 +108,7 @@ func createVolumesTestInput(pattern testpatterns.TestPattern, resource genericVo return volumesTestInput{ f: f, name: dInfo.Name, - config: dInfo.Config, + config: &dInfo.Config, fsGroup: fsGroup, resource: resource, tests: []framework.VolumeTest{ @@ -158,7 +158,7 @@ func (t *volumesTestSuite) execTest(driver TestDriver, pattern testpatterns.Test type volumesTestInput struct { f *framework.Framework name string - config framework.VolumeTestConfig + config *TestConfig fsGroup *int64 tests []framework.VolumeTest resource genericVolumeTestResource @@ -168,19 +168,20 @@ func testVolumes(input *volumesTestInput) { It("should be mountable", func() { f := input.f cs := f.ClientSet - defer framework.VolumeTestCleanup(f, input.config) + defer framework.VolumeTestCleanup(f, convertTestConfig(input.config)) skipPersistenceTest(input.resource.driver) volumeTest := input.tests - framework.InjectHtml(cs, input.config, volumeTest[0].Volume, volumeTest[0].ExpectedContent) - framework.TestVolumeClient(cs, input.config, input.fsGroup, input.tests) + config := convertTestConfig(input.config) + framework.InjectHtml(cs, config, volumeTest[0].Volume, volumeTest[0].ExpectedContent) + framework.TestVolumeClient(cs, config, input.fsGroup, input.tests) }) It("should allow exec of files on the volume", func() { f := input.f skipExecTest(input.resource.driver) - testScriptInPod(f, input.resource.volType, input.resource.volSource, input.config.NodeSelector) + testScriptInPod(f, input.resource.volType, input.resource.volSource, input.resource.driver.GetDriverInfo().Config.ClientNodeSelector) }) } From ac8ac8ee6c0efe31f5c4ea06fe0120963fdcdc68 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 27 Sep 2018 13:05:44 +0200 Subject: [PATCH 5/5] e2e: update bazel BUILD files Generated via hack/update-bazel.sh. --- test/e2e/storage/drivers/BUILD | 2 +- test/e2e/storage/testsuites/BUILD | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e/storage/drivers/BUILD b/test/e2e/storage/drivers/BUILD index 487e0f21866..da57c59f188 100644 --- a/test/e2e/storage/drivers/BUILD +++ b/test/e2e/storage/drivers/BUILD @@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ - "base.go", "csi.go", "csi_objects.go", "in_tree.go", @@ -24,6 +23,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/storage/testpatterns:go_default_library", + "//test/e2e/storage/testsuites:go_default_library", "//test/e2e/storage/utils:go_default_library", "//test/e2e/storage/vsphere:go_default_library", "//test/utils/image:go_default_library", diff --git a/test/e2e/storage/testsuites/BUILD b/test/e2e/storage/testsuites/BUILD index 800a19f6093..0ea47b1c697 100644 --- a/test/e2e/storage/testsuites/BUILD +++ b/test/e2e/storage/testsuites/BUILD @@ -4,8 +4,10 @@ go_library( name = "go_default_library", srcs = [ "base.go", + "driveroperations.go", "provisioning.go", "subpath.go", + "testdriver.go", "volume_io.go", "volumemode.go", "volumes.go", @@ -20,10 +22,10 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/framework:go_default_library", - "//test/e2e/storage/drivers:go_default_library", "//test/e2e/storage/testpatterns:go_default_library", "//test/e2e/storage/utils:go_default_library", "//test/utils/image:go_default_library",