From 725d5ebda46dc36c77fa52745bf47b5de9fe0cf5 Mon Sep 17 00:00:00 2001 From: David Zhu Date: Wed, 12 Sep 2018 11:05:29 -0700 Subject: [PATCH 1/3] Fix test framework to pass generic driver specific test specific resources. Fix GCE PD resource leak --- test/e2e/storage/drivers/base.go | 17 +-- test/e2e/storage/drivers/in_tree.go | 131 +++++++++++++--------- test/e2e/storage/testsuites/base.go | 10 +- test/e2e/storage/testsuites/subpath.go | 4 +- test/e2e/storage/testsuites/volume_io.go | 2 +- test/e2e/storage/testsuites/volumemode.go | 10 +- test/e2e/storage/testsuites/volumes.go | 2 +- 7 files changed, 102 insertions(+), 74 deletions(-) diff --git a/test/e2e/storage/drivers/base.go b/test/e2e/storage/drivers/base.go index f241ff750a0..e6d4109a9c6 100644 --- a/test/e2e/storage/drivers/base.go +++ b/test/e2e/storage/drivers/base.go @@ -44,9 +44,9 @@ type TestDriver interface { type PreprovisionedVolumeTestDriver interface { TestDriver // CreateVolume creates a pre-provisioned volume. - CreateVolume(testpatterns.TestVolType) + CreateVolume(testpatterns.TestVolType) interface{} // DeleteVolume deletes a volume that is created in CreateVolume - DeleteVolume(testpatterns.TestVolType) + DeleteVolume(testpatterns.TestVolType, interface{}) } // InlineVolumeTestDriver represents an interface for a TestDriver that supports InlineVolume @@ -55,7 +55,7 @@ type InlineVolumeTestDriver interface { // GetVolumeSource returns a volumeSource for inline volume. // It will set readOnly and fsType to the volumeSource, if TestDriver supports both of them. // It will return nil, if the TestDriver doesn't support either of the parameters. - GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource + GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource } // PreprovisionedPVTestDriver represents an interface for a TestDriver that supports PreprovisionedPV @@ -64,7 +64,7 @@ type PreprovisionedPVTestDriver interface { // GetPersistentVolumeSource returns a PersistentVolumeSource for pre-provisioned Persistent Volume. // It will set readOnly and fsType to the PersistentVolumeSource, if TestDriver supports both of them. // It will return nil, if the TestDriver doesn't support either of the parameters. - GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource + GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource } // DynamicPVTestDriver represents an interface for a TestDriver that supports DynamicPV @@ -104,30 +104,31 @@ func GetDriverNameWithFeatureTags(driver TestDriver) string { return fmt.Sprintf("[Driver: %s]%s", dInfo.Name, dInfo.FeatureTag) } -func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) { +func CreateVolume(driver TestDriver, volType testpatterns.TestVolType) interface{} { // Create Volume for test unless dynamicPV test switch volType { case testpatterns.InlineVolume: fallthrough case testpatterns.PreprovisionedPV: if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok { - pDriver.CreateVolume(volType) + return pDriver.CreateVolume(volType) } case testpatterns.DynamicPV: // No need to create volume default: framework.Failf("Invalid volType specified: %v", volType) } + return nil } -func DeleteVolume(driver TestDriver, volType testpatterns.TestVolType) { +func DeleteVolume(driver TestDriver, volType testpatterns.TestVolType, testResource interface{}) { // Delete Volume for test unless dynamicPV test switch volType { case testpatterns.InlineVolume: fallthrough case testpatterns.PreprovisionedPV: if pDriver, ok := driver.(PreprovisionedVolumeTestDriver); ok { - pDriver.DeleteVolume(volType) + pDriver.DeleteVolume(volType, testResource) } case testpatterns.DynamicPV: // No need to delete volume diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 5b5a9de2df4..359c07c7bc9 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -98,7 +98,7 @@ func (n *nfsDriver) GetDriverInfo() *DriverInfo { func (n *nfsDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } -func (n *nfsDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (n *nfsDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { return &v1.VolumeSource{ NFS: &v1.NFSVolumeSource{ Server: n.serverIP, @@ -108,7 +108,7 @@ func (n *nfsDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSour } } -func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { return &v1.PersistentVolumeSource{ NFS: &v1.NFSVolumeSource{ Server: n.serverIP, @@ -157,7 +157,7 @@ func (n *nfsDriver) CleanupDriver() { cs.RbacV1beta1().ClusterRoleBindings().Delete(clusterRoleBindingName, metav1.NewDeleteOptions(0)) } -func (n *nfsDriver) CreateVolume(volType testpatterns.TestVolType) { +func (n *nfsDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := n.driverInfo.Framework cs := f.ClientSet ns := f.Namespace @@ -175,9 +175,10 @@ func (n *nfsDriver) CreateVolume(volType testpatterns.TestVolType) { default: framework.Failf("Unsupported volType:%v is specified", volType) } + return nil } -func (n *nfsDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (n *nfsDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := n.driverInfo.Framework switch volType { @@ -232,7 +233,7 @@ func (g *glusterFSDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) } } -func (g *glusterFSDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (g *glusterFSDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { name := g.driverInfo.Config.Prefix + "-server" return &v1.VolumeSource{ Glusterfs: &v1.GlusterfsVolumeSource{ @@ -244,7 +245,7 @@ func (g *glusterFSDriver) GetVolumeSource(readOnly bool, fsType string) *v1.Volu } } -func (g *glusterFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (g *glusterFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { name := g.driverInfo.Config.Prefix + "-server" return &v1.PersistentVolumeSource{ Glusterfs: &v1.GlusterfsVolumeSource{ @@ -262,15 +263,16 @@ func (g *glusterFSDriver) CreateDriver() { func (g *glusterFSDriver) CleanupDriver() { } -func (g *glusterFSDriver) CreateVolume(volType testpatterns.TestVolType) { +func (g *glusterFSDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := g.driverInfo.Framework cs := f.ClientSet ns := f.Namespace g.driverInfo.Config, g.serverPod, g.serverIP = framework.NewGlusterfsServer(cs, ns.Name) + return nil } -func (g *glusterFSDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (g *glusterFSDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := g.driverInfo.Framework cs := f.ClientSet ns := f.Namespace @@ -331,7 +333,7 @@ func (i *iSCSIDriver) GetDriverInfo() *DriverInfo { func (i *iSCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } -func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { volSource := v1.VolumeSource{ ISCSI: &v1.ISCSIVolumeSource{ TargetPortal: i.serverIP + ":3260", @@ -347,7 +349,7 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSo return &volSource } -func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { pvSource := v1.PersistentVolumeSource{ ISCSI: &v1.ISCSIPersistentVolumeSource{ TargetPortal: i.serverIP + ":3260", @@ -368,15 +370,16 @@ func (i *iSCSIDriver) CreateDriver() { func (i *iSCSIDriver) CleanupDriver() { } -func (i *iSCSIDriver) CreateVolume(volType testpatterns.TestVolType) { +func (i *iSCSIDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := i.driverInfo.Framework cs := f.ClientSet ns := f.Namespace i.driverInfo.Config, i.serverPod, i.serverIP = framework.NewISCSIServer(cs, ns.Name) + return nil } -func (i *iSCSIDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (i *iSCSIDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := i.driverInfo.Framework framework.CleanUpVolumeServer(f, i.serverPod) @@ -423,7 +426,7 @@ func (r *rbdDriver) GetDriverInfo() *DriverInfo { func (r *rbdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } -func (r *rbdDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (r *rbdDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { volSource := v1.VolumeSource{ RBD: &v1.RBDVolumeSource{ CephMonitors: []string{r.serverIP}, @@ -442,7 +445,7 @@ func (r *rbdDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSour return &volSource } -func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { f := r.driverInfo.Framework ns := f.Namespace pvSource := v1.PersistentVolumeSource{ @@ -470,15 +473,16 @@ func (r *rbdDriver) CreateDriver() { func (r *rbdDriver) CleanupDriver() { } -func (r *rbdDriver) CreateVolume(volType testpatterns.TestVolType) { +func (r *rbdDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := r.driverInfo.Framework cs := f.ClientSet ns := f.Namespace r.driverInfo.Config, r.serverPod, r.secret, r.serverIP = framework.NewRBDServer(cs, ns.Name) + return nil } -func (r *rbdDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (r *rbdDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := r.driverInfo.Framework framework.CleanUpVolumeServerWithSecret(f, r.serverPod, r.secret) @@ -522,7 +526,7 @@ func (c *cephFSDriver) GetDriverInfo() *DriverInfo { func (c *cephFSDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } -func (c *cephFSDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (c *cephFSDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { return &v1.VolumeSource{ CephFS: &v1.CephFSVolumeSource{ Monitors: []string{c.serverIP + ":6789"}, @@ -535,7 +539,7 @@ func (c *cephFSDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeS } } -func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { f := c.driverInfo.Framework ns := f.Namespace @@ -558,15 +562,16 @@ func (c *cephFSDriver) CreateDriver() { func (c *cephFSDriver) CleanupDriver() { } -func (c *cephFSDriver) CreateVolume(volType testpatterns.TestVolType) { +func (c *cephFSDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := c.driverInfo.Framework cs := f.ClientSet ns := f.Namespace c.driverInfo.Config, c.serverPod, c.secret, c.serverIP = framework.NewRBDServer(cs, ns.Name) + return nil } -func (c *cephFSDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (c *cephFSDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := c.driverInfo.Framework framework.CleanUpVolumeServerWithSecret(f, c.serverPod, c.secret) @@ -606,7 +611,7 @@ func (h *hostPathDriver) GetDriverInfo() *DriverInfo { func (h *hostPathDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } -func (h *hostPathDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (h *hostPathDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { // hostPath doesn't support readOnly volume if readOnly { return nil @@ -624,7 +629,7 @@ func (h *hostPathDriver) CreateDriver() { func (h *hostPathDriver) CleanupDriver() { } -func (h *hostPathDriver) CreateVolume(volType testpatterns.TestVolType) { +func (h *hostPathDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := h.driverInfo.Framework cs := f.ClientSet @@ -632,9 +637,10 @@ func (h *hostPathDriver) CreateVolume(volType testpatterns.TestVolType) { nodes := framework.GetReadySchedulableNodesOrDie(cs) node := nodes.Items[rand.Intn(len(nodes.Items))] h.driverInfo.Config.ClientNodeName = node.Name + return nil } -func (h *hostPathDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (h *hostPathDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { } // HostPathSymlink @@ -674,7 +680,7 @@ func (h *hostPathSymlinkDriver) GetDriverInfo() *DriverInfo { func (h *hostPathSymlinkDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } -func (h *hostPathSymlinkDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (h *hostPathSymlinkDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { // hostPathSymlink doesn't support readOnly volume if readOnly { return nil @@ -692,7 +698,7 @@ func (h *hostPathSymlinkDriver) CreateDriver() { func (h *hostPathSymlinkDriver) CleanupDriver() { } -func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) { +func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := h.driverInfo.Framework cs := f.ClientSet @@ -753,9 +759,10 @@ func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) { err = framework.DeletePodWithWait(f, f.ClientSet, pod) Expect(err).ToNot(HaveOccurred(), "while deleting hostPath init pod") + return nil } -func (h *hostPathSymlinkDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (h *hostPathSymlinkDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := h.driverInfo.Framework cmd := fmt.Sprintf("rm -rf %v&& rm -rf %v", h.targetPath, h.sourcePath) @@ -803,7 +810,7 @@ func (e *emptydirDriver) GetDriverInfo() *DriverInfo { func (e *emptydirDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } -func (e *emptydirDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (e *emptydirDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { // emptydir doesn't support readOnly volume if readOnly { return nil @@ -813,10 +820,11 @@ func (e *emptydirDriver) GetVolumeSource(readOnly bool, fsType string) *v1.Volum } } -func (e *emptydirDriver) CreateVolume(volType testpatterns.TestVolType) { +func (e *emptydirDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { + return nil } -func (e *emptydirDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (e *emptydirDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { } func (e *emptydirDriver) CreateDriver() { @@ -868,7 +876,7 @@ func (c *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { framework.SkipUnlessProviderIs("openstack") } -func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { volSource := v1.VolumeSource{ Cinder: &v1.CinderVolumeSource{ VolumeID: c.volumeID, @@ -881,7 +889,7 @@ func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeS return &volSource } -func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { pvSource := v1.PersistentVolumeSource{ Cinder: &v1.CinderPersistentVolumeSource{ VolumeID: c.volumeID, @@ -912,7 +920,7 @@ func (c *cinderDriver) CreateDriver() { func (c *cinderDriver) CleanupDriver() { } -func (c *cinderDriver) CreateVolume(volType testpatterns.TestVolType) { +func (c *cinderDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := c.driverInfo.Framework ns := f.Namespace @@ -943,9 +951,10 @@ func (c *cinderDriver) CreateVolume(volType testpatterns.TestVolType) { } framework.Logf("Volume ID: %s", c.volumeID) Expect(c.volumeID).NotTo(Equal("")) + return nil } -func (c *cinderDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (c *cinderDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { deleteCinderVolume(c.volumeName) } @@ -1013,10 +1022,12 @@ func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } } -func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { + gtr, ok := testResource.(*gcePdTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to GCE PD Test Resource") volSource := v1.VolumeSource{ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ - PDName: g.volumeName, + PDName: gtr.volumeName, ReadOnly: readOnly, }, } @@ -1026,10 +1037,12 @@ func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSo return &volSource } -func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { + gtr, ok := testResource.(*gcePdTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to GCE PD Test Resource") pvSource := v1.PersistentVolumeSource{ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ - PDName: g.volumeName, + PDName: gtr.volumeName, ReadOnly: readOnly, }, } @@ -1057,7 +1070,11 @@ func (g *gcePdDriver) CreateDriver() { func (g *gcePdDriver) CleanupDriver() { } -func (g *gcePdDriver) CreateVolume(volType testpatterns.TestVolType) { +type gcePdTestResource struct { + volumeName string +} + +func (g *gcePdDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { if volType == testpatterns.InlineVolume { // PD will be created in framework.TestContext.CloudConfig.Zone zone, // so pods should be also scheduled there. @@ -1066,13 +1083,17 @@ func (g *gcePdDriver) CreateVolume(volType testpatterns.TestVolType) { } } By("creating a test gce pd volume") - var err error - g.volumeName, err = framework.CreatePDWithRetry() + vname, err := framework.CreatePDWithRetry() Expect(err).NotTo(HaveOccurred()) + return &gcePdTestResource{ + volumeName: vname, + } } -func (g *gcePdDriver) DeleteVolume(volType testpatterns.TestVolType) { - framework.DeletePDWithRetry(g.volumeName) +func (g *gcePdDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { + gtr, ok := testResource.(*gcePdTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to GCE PD Test Resource") + framework.DeletePDWithRetry(gtr.volumeName) } // vSphere @@ -1113,7 +1134,7 @@ func (v *vSphereDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { framework.SkipUnlessProviderIs("vsphere") } -func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { // vSphere driver doesn't seem to support readOnly volume // TODO: check if it is correct if readOnly { @@ -1130,7 +1151,7 @@ func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string) *v1.Volume return &volSource } -func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { // vSphere driver doesn't seem to support readOnly volume // TODO: check if it is correct if readOnly { @@ -1165,16 +1186,17 @@ func (v *vSphereDriver) CreateDriver() { func (v *vSphereDriver) CleanupDriver() { } -func (v *vSphereDriver) CreateVolume(volType testpatterns.TestVolType) { +func (v *vSphereDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := v.driverInfo.Framework vspheretest.Bootstrap(f) v.nodeInfo = vspheretest.GetReadySchedulableRandomNodeInfo() var err error v.volumePath, err = v.nodeInfo.VSphere.CreateVolume(&vspheretest.VolumeOptions{}, v.nodeInfo.DataCenterRef) Expect(err).NotTo(HaveOccurred()) + return nil } -func (v *vSphereDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (v *vSphereDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { v.nodeInfo.VSphere.DeleteVolume(v.volumePath, v.nodeInfo.DataCenterRef) } @@ -1216,7 +1238,7 @@ func (a *azureDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { framework.SkipUnlessProviderIs("azure") } -func (a *azureDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (a *azureDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { diskName := a.volumeName[(strings.LastIndex(a.volumeName, "/") + 1):] volSource := v1.VolumeSource{ @@ -1232,7 +1254,7 @@ func (a *azureDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSo return &volSource } -func (a *azureDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (a *azureDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { diskName := a.volumeName[(strings.LastIndex(a.volumeName, "/") + 1):] pvSource := v1.PersistentVolumeSource{ @@ -1266,14 +1288,15 @@ func (a *azureDriver) CreateDriver() { func (a *azureDriver) CleanupDriver() { } -func (a *azureDriver) CreateVolume(volType testpatterns.TestVolType) { +func (a *azureDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { By("creating a test azure disk volume") var err error a.volumeName, err = framework.CreatePDWithRetry() Expect(err).NotTo(HaveOccurred()) + return nil } -func (a *azureDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (a *azureDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { framework.DeletePDWithRetry(a.volumeName) } @@ -1319,7 +1342,7 @@ func (a *awsDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { // TODO: Fix authorization error in attach operation and uncomment below /* -func (a *awsDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSource { +func (a *awsDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { volSource := v1.VolumeSource{ AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ VolumeID: a.volumeName, @@ -1332,7 +1355,7 @@ func (a *awsDriver) GetVolumeSource(readOnly bool, fsType string) *v1.VolumeSour return &volSource } -func (a *awsDriver) GetPersistentVolumeSource(readOnly bool, fsType string) *v1.PersistentVolumeSource { +func (a *awsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { pvSource := v1.PersistentVolumeSource{ AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ VolumeID: a.volumeName, @@ -1366,14 +1389,14 @@ func (a *awsDriver) CleanupDriver() { // TODO: Fix authorization error in attach operation and uncomment below /* -func (a *awsDriver) CreateVolume(volType testpatterns.TestVolType) { +func (a *awsDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { By("creating a test aws volume") var err error a.volumeName, err = framework.CreatePDWithRetry() Expect(err).NotTo(HaveOccurred()) } -func (a *awsDriver) DeleteVolume(volType testpatterns.TestVolType) { +func (a *awsDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { framework.DeletePDWithRetry(a.volumeName) } */ diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 50b3a33ff1b..5df99d308aa 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -125,6 +125,8 @@ type genericVolumeTestResource struct { pvc *v1.PersistentVolumeClaim pv *v1.PersistentVolume sc *storagev1.StorageClass + + driverTestResource interface{} } var _ TestResource = &genericVolumeTestResource{} @@ -139,19 +141,19 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat volType := pattern.VolType // Create volume for pre-provisioned volume tests - drivers.CreateVolume(driver, volType) + r.driverTestResource = drivers.CreateVolume(driver, volType) switch volType { case testpatterns.InlineVolume: framework.Logf("Creating resource for inline volume") if iDriver, ok := driver.(drivers.InlineVolumeTestDriver); ok { - r.volSource = iDriver.GetVolumeSource(false, fsType) + r.volSource = iDriver.GetVolumeSource(false, fsType, r.driverTestResource) r.volType = dInfo.Name } case testpatterns.PreprovisionedPV: framework.Logf("Creating resource for pre-provisioned PV") if pDriver, ok := driver.(drivers.PreprovisionedPVTestDriver); ok { - pvSource := pDriver.GetPersistentVolumeSource(false, fsType) + pvSource := pDriver.GetPersistentVolumeSource(false, fsType, r.driverTestResource) if pvSource != nil { r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, false) } @@ -202,7 +204,7 @@ func (r *genericVolumeTestResource) cleanupResource(driver drivers.TestDriver, p } // Cleanup volume for pre-provisioned volume tests - drivers.DeleteVolume(driver, volType) + drivers.DeleteVolume(driver, volType, r.driverTestResource) } func createVolumeSourceWithPVCPV( diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index 5c662d465da..8c04a908798 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -110,7 +110,7 @@ func (s *subPathTestSuite) execTest(driver drivers.TestDriver, pattern testpatte needsCleanup = true // Setup test resource for driver and testpattern - resource := subPathTestResource{} + resource = subPathTestResource{} resource.setupResource(driver, pattern) // Create test input @@ -151,7 +151,7 @@ func (s *subPathTestResource) setupResource(driver drivers.TestDriver, pattern t switch volType { case testpatterns.InlineVolume: if iDriver, ok := driver.(drivers.InlineVolumeTestDriver); ok { - s.roVolSource = iDriver.GetVolumeSource(true, fsType) + s.roVolSource = iDriver.GetVolumeSource(true, fsType, s.genericVolumeTestResource.driverTestResource) } case testpatterns.PreprovisionedPV: s.roVolSource = &v1.VolumeSource{ diff --git a/test/e2e/storage/testsuites/volume_io.go b/test/e2e/storage/testsuites/volume_io.go index 497151cbb6d..82cb0eaf4b0 100644 --- a/test/e2e/storage/testsuites/volume_io.go +++ b/test/e2e/storage/testsuites/volume_io.go @@ -121,7 +121,7 @@ func (t *volumeIOTestSuite) execTest(driver drivers.TestDriver, pattern testpatt needsCleanup = true // Setup test resource for driver and testpattern - resource := genericVolumeTestResource{} + resource = genericVolumeTestResource{} resource.setupResource(driver, pattern) // Create test input diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index e8f740ea3af..cd9992fe153 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -124,7 +124,7 @@ func (t *volumeModeTestSuite) execTest(driver drivers.TestDriver, pattern testpa needsCleanup = true // Setup test resource for driver and testpattern - resource := volumeModeTestResource{} + resource = volumeModeTestResource{} resource.setupResource(driver, pattern) // Create test input @@ -147,6 +147,8 @@ type volumeModeTestResource struct { sc *storagev1.StorageClass pvc *v1.PersistentVolumeClaim pv *v1.PersistentVolume + + driverTestResource interface{} } var _ TestResource = &volumeModeTestResource{} @@ -167,7 +169,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter ) // Create volume for pre-provisioned volume tests - drivers.CreateVolume(driver, volType) + s.driverTestResource = drivers.CreateVolume(driver, volType) switch volType { case testpatterns.PreprovisionedPV: @@ -177,7 +179,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter scName = fmt.Sprintf("%s-%s-sc-for-file", ns.Name, dInfo.Name) } if pDriver, ok := driver.(drivers.PreprovisionedPVTestDriver); ok { - pvSource = pDriver.GetPersistentVolumeSource(false, fsType) + pvSource = pDriver.GetPersistentVolumeSource(false, fsType, s.driverTestResource) if pvSource == nil { framework.Skipf("Driver %q does not define PersistentVolumeSource - skipping", dInfo.Name) } @@ -223,7 +225,7 @@ func (s *volumeModeTestResource) cleanupResource(driver drivers.TestDriver, patt } // Cleanup volume for pre-provisioned volume tests - drivers.DeleteVolume(driver, volType) + drivers.DeleteVolume(driver, volType, s.driverTestResource) } type volumeModeTestInput struct { diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index c7c65110804..ce8e2093e86 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -122,7 +122,7 @@ func (t *volumesTestSuite) execTest(driver drivers.TestDriver, pattern testpatte needsCleanup = true // Setup test resource for driver and testpattern - resource := genericVolumeTestResource{} + resource = genericVolumeTestResource{} resource.setupResource(driver, pattern) // Create test input From 5155e1b5d953c3ce4710f78a76fd83515865b5e4 Mon Sep 17 00:00:00 2001 From: David Zhu Date: Wed, 12 Sep 2018 13:40:47 -0700 Subject: [PATCH 2/3] Test Resource fix for all non-GCE In-tree drivers --- test/e2e/storage/drivers/in_tree.go | 317 ++++++++++++++++++++-------- 1 file changed, 224 insertions(+), 93 deletions(-) diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 359c07c7bc9..953917b3e76 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -61,14 +61,17 @@ import ( // NFS type nfsDriver struct { - serverIP string - serverPod *v1.Pod externalProvisionerPod *v1.Pod externalPluginName string driverInfo DriverInfo } +type nfsTestResource struct { + serverIP string + serverPod *v1.Pod +} + var _ TestDriver = &nfsDriver{} var _ PreprovisionedVolumeTestDriver = &nfsDriver{} var _ InlineVolumeTestDriver = &nfsDriver{} @@ -99,9 +102,11 @@ func (n *nfsDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } func (n *nfsDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { + ntr, ok := testResource.(*nfsTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to NFS Test Resource") return &v1.VolumeSource{ NFS: &v1.NFSVolumeSource{ - Server: n.serverIP, + Server: ntr.serverIP, Path: "/", ReadOnly: readOnly, }, @@ -109,9 +114,11 @@ func (n *nfsDriver) GetVolumeSource(readOnly bool, fsType string, testResource i } func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { + ntr, ok := testResource.(*nfsTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to NFS Test Resource") return &v1.PersistentVolumeSource{ NFS: &v1.NFSVolumeSource{ - Server: n.serverIP, + Server: ntr.serverIP, Path: "/", ReadOnly: readOnly, }, @@ -169,7 +176,12 @@ func (n *nfsDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { case testpatterns.InlineVolume: fallthrough case testpatterns.PreprovisionedPV: - n.driverInfo.Config, n.serverPod, n.serverIP = framework.NewNFSServer(cs, ns.Name, []string{}) + config, serverPod, serverIP := framework.NewNFSServer(cs, ns.Name, []string{}) + n.driverInfo.Config = config + return &nfsTestResource{ + serverIP: serverIP, + serverPod: serverPod, + } case testpatterns.DynamicPV: // Do nothing default: @@ -181,11 +193,14 @@ func (n *nfsDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { func (n *nfsDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := n.driverInfo.Framework + ntr, ok := testResource.(*nfsTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to NFS Test Resource") + switch volType { case testpatterns.InlineVolume: fallthrough case testpatterns.PreprovisionedPV: - framework.CleanUpVolumeServer(f, n.serverPod) + framework.CleanUpVolumeServer(f, ntr.serverPod) case testpatterns.DynamicPV: // Do nothing default: @@ -195,12 +210,14 @@ func (n *nfsDriver) DeleteVolume(volType testpatterns.TestVolType, testResource // Gluster type glusterFSDriver struct { - serverIP string - serverPod *v1.Pod - driverInfo DriverInfo } +type glusterTestResource struct { + prefix string + serverPod *v1.Pod +} + var _ TestDriver = &glusterFSDriver{} var _ PreprovisionedVolumeTestDriver = &glusterFSDriver{} var _ InlineVolumeTestDriver = &glusterFSDriver{} @@ -234,7 +251,10 @@ func (g *glusterFSDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) } func (g *glusterFSDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { - name := g.driverInfo.Config.Prefix + "-server" + gtr, ok := testResource.(*glusterTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Gluster Test Resource") + + name := gtr.prefix + "-server" return &v1.VolumeSource{ Glusterfs: &v1.GlusterfsVolumeSource{ EndpointsName: name, @@ -246,7 +266,10 @@ func (g *glusterFSDriver) GetVolumeSource(readOnly bool, fsType string, testReso } func (g *glusterFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { - name := g.driverInfo.Config.Prefix + "-server" + gtr, ok := testResource.(*glusterTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Gluster Test Resource") + + name := gtr.prefix + "-server" return &v1.PersistentVolumeSource{ Glusterfs: &v1.GlusterfsVolumeSource{ EndpointsName: name, @@ -268,8 +291,12 @@ func (g *glusterFSDriver) CreateVolume(volType testpatterns.TestVolType) interfa cs := f.ClientSet ns := f.Namespace - g.driverInfo.Config, g.serverPod, g.serverIP = framework.NewGlusterfsServer(cs, ns.Name) - return nil + config, serverPod, _ := framework.NewGlusterfsServer(cs, ns.Name) + g.driverInfo.Config = config + return &glusterTestResource{ + prefix: config.Prefix, + serverPod: serverPod, + } } func (g *glusterFSDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { @@ -277,15 +304,18 @@ func (g *glusterFSDriver) DeleteVolume(volType testpatterns.TestVolType, testRes cs := f.ClientSet ns := f.Namespace - name := g.driverInfo.Config.Prefix + "-server" + gtr, ok := testResource.(*glusterTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Gluster Test Resource") + + name := gtr.prefix + "-server" framework.Logf("Deleting Gluster endpoints %q...", name) err := cs.CoreV1().Endpoints(ns.Name).Delete(name, nil) if err != nil { framework.Failf("Gluster delete endpoints failed: %v", err) } - framework.Logf("Deleting Gluster server pod %q...", g.serverPod.Name) - err = framework.DeletePodWithWait(f, cs, g.serverPod) + framework.Logf("Deleting Gluster server pod %q...", gtr.serverPod.Name) + err = framework.DeletePodWithWait(f, cs, gtr.serverPod) if err != nil { framework.Failf("Gluster server pod delete failed: %v", err) } @@ -294,11 +324,12 @@ func (g *glusterFSDriver) DeleteVolume(volType testpatterns.TestVolType, testRes // iSCSI // The iscsiadm utility and iscsi target kernel modules must be installed on all nodes. type iSCSIDriver struct { - serverIP string - serverPod *v1.Pod - driverInfo DriverInfo } +type iSCSITestResource struct { + serverPod *v1.Pod + serverIP string +} var _ TestDriver = &iSCSIDriver{} var _ PreprovisionedVolumeTestDriver = &iSCSIDriver{} @@ -334,9 +365,12 @@ func (i *iSCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { + itr, ok := testResource.(*iSCSITestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to iSCSI Test Resource") + volSource := v1.VolumeSource{ ISCSI: &v1.ISCSIVolumeSource{ - TargetPortal: i.serverIP + ":3260", + TargetPortal: itr.serverIP + ":3260", // from test/images/volume/iscsi/initiatorname.iscsi IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c", Lun: 0, @@ -350,9 +384,12 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, testResource } func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { + itr, ok := testResource.(*iSCSITestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to iSCSI Test Resource") + pvSource := v1.PersistentVolumeSource{ ISCSI: &v1.ISCSIPersistentVolumeSource{ - TargetPortal: i.serverIP + ":3260", + TargetPortal: itr.serverIP + ":3260", IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c", Lun: 0, ReadOnly: readOnly, @@ -375,25 +412,34 @@ func (i *iSCSIDriver) CreateVolume(volType testpatterns.TestVolType) interface{} cs := f.ClientSet ns := f.Namespace - i.driverInfo.Config, i.serverPod, i.serverIP = framework.NewISCSIServer(cs, ns.Name) - return nil + config, serverPod, serverIP := framework.NewISCSIServer(cs, ns.Name) + i.driverInfo.Config = config + return &iSCSITestResource{ + serverPod: serverPod, + serverIP: serverIP, + } } func (i *iSCSIDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := i.driverInfo.Framework - framework.CleanUpVolumeServer(f, i.serverPod) + itr, ok := testResource.(*iSCSITestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to iSCSI Test Resource") + + framework.CleanUpVolumeServer(f, itr.serverPod) } // Ceph RBD type rbdDriver struct { - serverIP string - serverPod *v1.Pod - secret *v1.Secret - driverInfo DriverInfo } +type rbdTestResource struct { + serverPod *v1.Pod + serverIP string + secret *v1.Secret +} + var _ TestDriver = &rbdDriver{} var _ PreprovisionedVolumeTestDriver = &rbdDriver{} var _ InlineVolumeTestDriver = &rbdDriver{} @@ -427,14 +473,17 @@ func (r *rbdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } func (r *rbdDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { + rtr, ok := testResource.(*rbdTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to RBD Test Resource") + volSource := v1.VolumeSource{ RBD: &v1.RBDVolumeSource{ - CephMonitors: []string{r.serverIP}, + CephMonitors: []string{rtr.serverIP}, RBDPool: "rbd", RBDImage: "foo", RadosUser: "admin", SecretRef: &v1.LocalObjectReference{ - Name: r.secret.Name, + Name: rtr.secret.Name, }, ReadOnly: readOnly, }, @@ -448,14 +497,18 @@ func (r *rbdDriver) GetVolumeSource(readOnly bool, fsType string, testResource i func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { f := r.driverInfo.Framework ns := f.Namespace + + rtr, ok := testResource.(*rbdTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to RBD Test Resource") + pvSource := v1.PersistentVolumeSource{ RBD: &v1.RBDPersistentVolumeSource{ - CephMonitors: []string{r.serverIP}, + CephMonitors: []string{rtr.serverIP}, RBDPool: "rbd", RBDImage: "foo", RadosUser: "admin", SecretRef: &v1.SecretReference{ - Name: r.secret.Name, + Name: rtr.secret.Name, Namespace: ns.Name, }, ReadOnly: readOnly, @@ -478,14 +531,22 @@ func (r *rbdDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { cs := f.ClientSet ns := f.Namespace - r.driverInfo.Config, r.serverPod, r.secret, r.serverIP = framework.NewRBDServer(cs, ns.Name) - return nil + config, serverPod, secret, serverIP := framework.NewRBDServer(cs, ns.Name) + r.driverInfo.Config = config + return &rbdTestResource{ + serverPod: serverPod, + serverIP: serverIP, + secret: secret, + } } func (r *rbdDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := r.driverInfo.Framework - framework.CleanUpVolumeServerWithSecret(f, r.serverPod, r.secret) + rtr, ok := testResource.(*rbdTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to RBD Test Resource") + + framework.CleanUpVolumeServerWithSecret(f, rtr.serverPod, rtr.secret) } // Ceph @@ -497,6 +558,12 @@ type cephFSDriver struct { driverInfo DriverInfo } +type cephTestResource struct { + serverPod *v1.Pod + serverIP string + secret *v1.Secret +} + var _ TestDriver = &cephFSDriver{} var _ PreprovisionedVolumeTestDriver = &cephFSDriver{} var _ InlineVolumeTestDriver = &cephFSDriver{} @@ -527,12 +594,15 @@ func (c *cephFSDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } func (c *cephFSDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { + ctr, ok := testResource.(*cephTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Ceph Test Resource") + return &v1.VolumeSource{ CephFS: &v1.CephFSVolumeSource{ - Monitors: []string{c.serverIP + ":6789"}, + Monitors: []string{ctr.serverIP + ":6789"}, User: "kube", SecretRef: &v1.LocalObjectReference{ - Name: c.secret.Name, + Name: ctr.secret.Name, }, ReadOnly: readOnly, }, @@ -543,12 +613,15 @@ func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, t f := c.driverInfo.Framework ns := f.Namespace + ctr, ok := testResource.(*cephTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Ceph Test Resource") + return &v1.PersistentVolumeSource{ CephFS: &v1.CephFSPersistentVolumeSource{ - Monitors: []string{c.serverIP + ":6789"}, + Monitors: []string{ctr.serverIP + ":6789"}, User: "kube", SecretRef: &v1.SecretReference{ - Name: c.secret.Name, + Name: ctr.secret.Name, Namespace: ns.Name, }, ReadOnly: readOnly, @@ -567,14 +640,22 @@ func (c *cephFSDriver) CreateVolume(volType testpatterns.TestVolType) interface{ cs := f.ClientSet ns := f.Namespace - c.driverInfo.Config, c.serverPod, c.secret, c.serverIP = framework.NewRBDServer(cs, ns.Name) - return nil + config, serverPod, secret, serverIP := framework.NewRBDServer(cs, ns.Name) + c.driverInfo.Config = config + return &cephTestResource{ + serverPod: serverPod, + serverIP: serverIP, + secret: secret, + } } func (c *cephFSDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := c.driverInfo.Framework - framework.CleanUpVolumeServerWithSecret(f, c.serverPod, c.secret) + ctr, ok := testResource.(*cephTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Ceph Test Resource") + + framework.CleanUpVolumeServerWithSecret(f, ctr.serverPod, ctr.secret) } // Hostpath @@ -645,14 +726,17 @@ func (h *hostPathDriver) DeleteVolume(volType testpatterns.TestVolType, testReso // HostPathSymlink type hostPathSymlinkDriver struct { - node v1.Node - sourcePath string - targetPath string - prepPod *v1.Pod + node v1.Node driverInfo DriverInfo } +type hostPathSymlinkTestResource struct { + targetPath string + sourcePath string + prepPod *v1.Pod +} + var _ TestDriver = &hostPathSymlinkDriver{} var _ PreprovisionedVolumeTestDriver = &hostPathSymlinkDriver{} var _ InlineVolumeTestDriver = &hostPathSymlinkDriver{} @@ -681,13 +765,16 @@ func (h *hostPathSymlinkDriver) SkipUnsupportedTest(pattern testpatterns.TestPat } func (h *hostPathSymlinkDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { + htr, ok := testResource.(*hostPathSymlinkTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Hostpath Symlink Test Resource") + // hostPathSymlink doesn't support readOnly volume if readOnly { return nil } return &v1.VolumeSource{ HostPath: &v1.HostPathVolumeSource{ - Path: h.targetPath, + Path: htr.targetPath, }, } } @@ -702,8 +789,8 @@ func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) i f := h.driverInfo.Framework cs := f.ClientSet - h.sourcePath = fmt.Sprintf("/tmp/%v", f.Namespace.Name) - h.targetPath = fmt.Sprintf("/tmp/%v-link", f.Namespace.Name) + sourcePath := fmt.Sprintf("/tmp/%v", f.Namespace.Name) + targetPath := fmt.Sprintf("/tmp/%v-link", f.Namespace.Name) volumeName := "test-volume" // pods should be scheduled on the node @@ -711,11 +798,11 @@ func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) i node := nodes.Items[rand.Intn(len(nodes.Items))] h.driverInfo.Config.ClientNodeName = node.Name - cmd := fmt.Sprintf("mkdir %v -m 777 && ln -s %v %v", h.sourcePath, h.sourcePath, h.targetPath) + cmd := fmt.Sprintf("mkdir %v -m 777 && ln -s %v %v", sourcePath, sourcePath, targetPath) privileged := true // Launch pod to initialize hostPath directory and symlink - h.prepPod = &v1.Pod{ + prepPod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("hostpath-symlink-prep-%s", f.Namespace.Name), }, @@ -751,7 +838,7 @@ func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) i }, } // h.prepPod will be reused in cleanupDriver. - pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(h.prepPod) + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(prepPod) Expect(err).ToNot(HaveOccurred(), "while creating hostPath init pod") err = framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, pod.Namespace) @@ -759,16 +846,23 @@ func (h *hostPathSymlinkDriver) CreateVolume(volType testpatterns.TestVolType) i err = framework.DeletePodWithWait(f, f.ClientSet, pod) Expect(err).ToNot(HaveOccurred(), "while deleting hostPath init pod") - return nil + return &hostPathSymlinkTestResource{ + sourcePath: sourcePath, + targetPath: targetPath, + prepPod: prepPod, + } } func (h *hostPathSymlinkDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { f := h.driverInfo.Framework - cmd := fmt.Sprintf("rm -rf %v&& rm -rf %v", h.targetPath, h.sourcePath) - h.prepPod.Spec.Containers[0].Command = []string{"/bin/sh", "-ec", cmd} + htr, ok := testResource.(*hostPathSymlinkTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Hostpath Symlink Test Resource") - pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(h.prepPod) + cmd := fmt.Sprintf("rm -rf %v&& rm -rf %v", htr.targetPath, htr.sourcePath) + htr.prepPod.Spec.Containers[0].Command = []string{"/bin/sh", "-ec", cmd} + + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(htr.prepPod) Expect(err).ToNot(HaveOccurred(), "while creating hostPath teardown pod") err = framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, pod.Namespace) @@ -839,10 +933,12 @@ func (e *emptydirDriver) CleanupDriver() { // and that the usual OpenStack authentication env. variables are set // (OS_USERNAME, OS_PASSWORD, OS_TENANT_NAME at least). type cinderDriver struct { + driverInfo DriverInfo +} + +type cinderTestResource struct { volumeName string volumeID string - - driverInfo DriverInfo } var _ TestDriver = &cinderDriver{} @@ -877,9 +973,12 @@ func (c *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { + ctr, ok := testResource.(*cinderTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Cinder Test Resource") + volSource := v1.VolumeSource{ Cinder: &v1.CinderVolumeSource{ - VolumeID: c.volumeID, + VolumeID: ctr.volumeID, ReadOnly: readOnly, }, } @@ -890,9 +989,12 @@ func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string, testResourc } func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { + ctr, ok := testResource.(*cinderTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Cinder Test Resource") + pvSource := v1.PersistentVolumeSource{ Cinder: &v1.CinderPersistentVolumeSource{ - VolumeID: c.volumeID, + VolumeID: ctr.volumeID, ReadOnly: readOnly, }, } @@ -925,9 +1027,9 @@ func (c *cinderDriver) CreateVolume(volType testpatterns.TestVolType) interface{ ns := f.Namespace // We assume that namespace.Name is a random string - c.volumeName = ns.Name + volumeName := ns.Name By("creating a test Cinder volume") - output, err := exec.Command("cinder", "create", "--display-name="+c.volumeName, "1").CombinedOutput() + output, err := exec.Command("cinder", "create", "--display-name="+volumeName, "1").CombinedOutput() outputString := string(output[:]) framework.Logf("cinder output:\n%s", outputString) Expect(err).NotTo(HaveOccurred()) @@ -937,7 +1039,7 @@ func (c *cinderDriver) CreateVolume(volType testpatterns.TestVolType) interface{ // | availability_zone | nova | // ... // | id | 1d6ff08f-5d1c-41a4-ad72-4ef872cae685 | - c.volumeID = "" + volumeID := "" for _, line := range strings.Split(outputString, "\n") { fields := strings.Fields(line) if len(fields) != 5 { @@ -946,16 +1048,22 @@ func (c *cinderDriver) CreateVolume(volType testpatterns.TestVolType) interface{ if fields[1] != "id" { continue } - c.volumeID = fields[3] + volumeID = fields[3] break } - framework.Logf("Volume ID: %s", c.volumeID) - Expect(c.volumeID).NotTo(Equal("")) - return nil + framework.Logf("Volume ID: %s", volumeID) + Expect(volumeID).NotTo(Equal("")) + return &cinderTestResource{ + volumeName: volumeName, + volumeID: volumeID, + } } func (c *cinderDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - deleteCinderVolume(c.volumeName) + ctr, ok := testResource.(*cinderTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Cinder Test Resource") + + deleteCinderVolume(ctr.volumeName) } func deleteCinderVolume(name string) error { @@ -980,11 +1088,13 @@ func deleteCinderVolume(name string) error { // GCE type gcePdDriver struct { - volumeName string - driverInfo DriverInfo } +type gcePdTestResource struct { + volumeName string +} + var _ TestDriver = &gcePdDriver{} var _ PreprovisionedVolumeTestDriver = &gcePdDriver{} var _ InlineVolumeTestDriver = &gcePdDriver{} @@ -1070,10 +1180,6 @@ func (g *gcePdDriver) CreateDriver() { func (g *gcePdDriver) CleanupDriver() { } -type gcePdTestResource struct { - volumeName string -} - func (g *gcePdDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { if volType == testpatterns.InlineVolume { // PD will be created in framework.TestContext.CloudConfig.Zone zone, @@ -1098,10 +1204,12 @@ func (g *gcePdDriver) DeleteVolume(volType testpatterns.TestVolType, testResourc // vSphere type vSphereDriver struct { + driverInfo DriverInfo +} + +type vSphereTestResource struct { volumePath string nodeInfo *vspheretest.NodeInfo - - driverInfo DriverInfo } var _ TestDriver = &vSphereDriver{} @@ -1135,6 +1243,9 @@ func (v *vSphereDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { + vtr, ok := testResource.(*vSphereTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to vSphere Test Resource") + // vSphere driver doesn't seem to support readOnly volume // TODO: check if it is correct if readOnly { @@ -1142,7 +1253,7 @@ func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, testResour } volSource := v1.VolumeSource{ VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{ - VolumePath: v.volumePath, + VolumePath: vtr.volumePath, }, } if fsType != "" { @@ -1152,6 +1263,9 @@ func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, testResour } func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { + vtr, ok := testResource.(*vSphereTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to vSphere Test Resource") + // vSphere driver doesn't seem to support readOnly volume // TODO: check if it is correct if readOnly { @@ -1159,7 +1273,7 @@ func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, } pvSource := v1.PersistentVolumeSource{ VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{ - VolumePath: v.volumePath, + VolumePath: vtr.volumePath, }, } if fsType != "" { @@ -1189,24 +1303,31 @@ func (v *vSphereDriver) CleanupDriver() { func (v *vSphereDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { f := v.driverInfo.Framework vspheretest.Bootstrap(f) - v.nodeInfo = vspheretest.GetReadySchedulableRandomNodeInfo() - var err error - v.volumePath, err = v.nodeInfo.VSphere.CreateVolume(&vspheretest.VolumeOptions{}, v.nodeInfo.DataCenterRef) + nodeInfo := vspheretest.GetReadySchedulableRandomNodeInfo() + volumePath, err := nodeInfo.VSphere.CreateVolume(&vspheretest.VolumeOptions{}, nodeInfo.DataCenterRef) Expect(err).NotTo(HaveOccurred()) - return nil + return &vSphereTestResource{ + volumePath: volumePath, + nodeInfo: nodeInfo, + } } func (v *vSphereDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - v.nodeInfo.VSphere.DeleteVolume(v.volumePath, v.nodeInfo.DataCenterRef) + vtr, ok := testResource.(*vSphereTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to vSphere Test Resource") + + vtr.nodeInfo.VSphere.DeleteVolume(vtr.volumePath, vtr.nodeInfo.DataCenterRef) } // Azure type azureDriver struct { - volumeName string - driverInfo DriverInfo } +type azureTestResource struct { + volumeName string +} + var _ TestDriver = &azureDriver{} var _ PreprovisionedVolumeTestDriver = &azureDriver{} var _ InlineVolumeTestDriver = &azureDriver{} @@ -1239,12 +1360,15 @@ func (a *azureDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { } func (a *azureDriver) GetVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.VolumeSource { - diskName := a.volumeName[(strings.LastIndex(a.volumeName, "/") + 1):] + atr, ok := testResource.(*azureTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Azure Test Resource") + + diskName := atr.volumeName[(strings.LastIndex(atr.volumeName, "/") + 1):] volSource := v1.VolumeSource{ AzureDisk: &v1.AzureDiskVolumeSource{ DiskName: diskName, - DataDiskURI: a.volumeName, + DataDiskURI: atr.volumeName, ReadOnly: &readOnly, }, } @@ -1255,12 +1379,15 @@ func (a *azureDriver) GetVolumeSource(readOnly bool, fsType string, testResource } func (a *azureDriver) GetPersistentVolumeSource(readOnly bool, fsType string, testResource interface{}) *v1.PersistentVolumeSource { - diskName := a.volumeName[(strings.LastIndex(a.volumeName, "/") + 1):] + atr, ok := testResource.(*azureTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Azure Test Resource") + + diskName := atr.volumeName[(strings.LastIndex(atr.volumeName, "/") + 1):] pvSource := v1.PersistentVolumeSource{ AzureDisk: &v1.AzureDiskVolumeSource{ DiskName: diskName, - DataDiskURI: a.volumeName, + DataDiskURI: atr.volumeName, ReadOnly: &readOnly, }, } @@ -1290,14 +1417,18 @@ func (a *azureDriver) CleanupDriver() { func (a *azureDriver) CreateVolume(volType testpatterns.TestVolType) interface{} { By("creating a test azure disk volume") - var err error - a.volumeName, err = framework.CreatePDWithRetry() + volumeName, err := framework.CreatePDWithRetry() Expect(err).NotTo(HaveOccurred()) - return nil + return &azureTestResource{ + volumeName: volumeName, + } } func (a *azureDriver) DeleteVolume(volType testpatterns.TestVolType, testResource interface{}) { - framework.DeletePDWithRetry(a.volumeName) + atr, ok := testResource.(*azureTestResource) + Expect(ok).To(BeTrue(), "Failed to cast test resource to Azure Test Resource") + + framework.DeletePDWithRetry(atr.volumeName) } // AWS From f3238d238d7093cf912104786cd956a15cc6f4a6 Mon Sep 17 00:00:00 2001 From: David Zhu Date: Wed, 12 Sep 2018 14:04:06 -0700 Subject: [PATCH 3/3] Change GCEPD Standard test disk to PD_STANDARD and 2GB --- test/e2e/framework/pv_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/framework/pv_util.go b/test/e2e/framework/pv_util.go index 8b14bedf3b6..a29e94ba9d5 100644 --- a/test/e2e/framework/pv_util.go +++ b/test/e2e/framework/pv_util.go @@ -721,7 +721,7 @@ func createPD(zone string) (string, error) { } tags := map[string]string{} - err = gceCloud.CreateDisk(pdName, gcecloud.DiskTypeSSD, zone, 10 /* sizeGb */, tags) + err = gceCloud.CreateDisk(pdName, gcecloud.DiskTypeStandard, zone, 2 /* sizeGb */, tags) if err != nil { return "", err }