From 7171f9a7f4665e2fb085db6cb1e288f79cb70c0d Mon Sep 17 00:00:00 2001 From: clarklee92 Date: Tue, 31 Dec 2019 18:43:08 +0800 Subject: [PATCH] move funs of framework/volume to e2e/storage Signed-off-by: clarklee92 --- test/e2e/framework/volume/BUILD | 1 - test/e2e/framework/volume/fixtures.go | 132 +----------------- test/e2e/storage/drivers/in_tree.go | 108 +++++++++++++- .../nfs_persistent_volume-disruptive.go | 17 ++- test/e2e/storage/testsuites/volumes.go | 22 ++- 5 files changed, 141 insertions(+), 139 deletions(-) diff --git a/test/e2e/framework/volume/BUILD b/test/e2e/framework/volume/BUILD index 3d1e2449af7..171c9d61ed6 100644 --- a/test/e2e/framework/volume/BUILD +++ b/test/e2e/framework/volume/BUILD @@ -14,7 +14,6 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/pod:go_default_library", - "//test/e2e/framework/pv:go_default_library", "//test/e2e/storage/utils:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", diff --git a/test/e2e/framework/volume/fixtures.go b/test/e2e/framework/volume/fixtures.go index cf57ed2a1a0..35be9d04d96 100644 --- a/test/e2e/framework/volume/fixtures.go +++ b/test/e2e/framework/volume/fixtures.go @@ -52,7 +52,6 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2epv "k8s.io/kubernetes/test/e2e/framework/pv" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -84,9 +83,6 @@ const ( // PodCleanupTimeout is a waiting period for pod to be cleaned up and unmount its volumes so we // don't tear down containers with NFS/Ceph/Gluster server too early. PodCleanupTimeout = 20 * time.Second - - // Template for iSCSI IQN. - iSCSIIQNTemplate = "iqn.2003-01.io.k8s:e2e.%s" ) // SizeRange encapsulates a range of sizes specified as minimum and maximum quantity strings @@ -208,74 +204,11 @@ func NewGlusterfsServer(cs clientset.Interface, namespace string) (config TestCo return config, pod, ip } -// NewISCSIServer is an iSCSI-specific wrapper for CreateStorageServer. -func NewISCSIServer(cs clientset.Interface, namespace string) (config TestConfig, pod *v1.Pod, ip, iqn string) { - // Generate cluster-wide unique IQN - iqn = fmt.Sprintf(iSCSIIQNTemplate, namespace) - config = TestConfig{ - Namespace: namespace, - Prefix: "iscsi", - ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer), - ServerArgs: []string{iqn}, - ServerVolumes: map[string]string{ - // iSCSI container needs to insert modules from the host - "/lib/modules": "/lib/modules", - // iSCSI container needs to configure kernel - "/sys/kernel": "/sys/kernel", - // iSCSI source "block devices" must be available on the host - "/srv/iscsi": "/srv/iscsi", - }, - ServerReadyMessage: "iscsi target started", - ServerHostNetwork: true, - } - pod, ip = CreateStorageServer(cs, config) - // Make sure the client runs on the same node as server so we don't need to open any firewalls. - config.ClientNodeName = pod.Spec.NodeName - return config, pod, ip, iqn -} - -// NewRBDServer is a CephRBD-specific wrapper for CreateStorageServer. -func NewRBDServer(cs clientset.Interface, namespace string) (config TestConfig, pod *v1.Pod, secret *v1.Secret, ip string) { - config = TestConfig{ - Namespace: namespace, - Prefix: "rbd", - ServerImage: imageutils.GetE2EImage(imageutils.VolumeRBDServer), - ServerPorts: []int{6789}, - ServerVolumes: map[string]string{ - "/lib/modules": "/lib/modules", - }, - ServerReadyMessage: "Ceph is ready", - } - pod, ip = CreateStorageServer(cs, config) - // create secrets for the server - secret = &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: config.Prefix + "-secret", - }, - Data: map[string][]byte{ - // from test/images/volumes-tester/rbd/keyring - "key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="), - }, - Type: "kubernetes.io/rbd", - } - - secret, err := cs.CoreV1().Secrets(config.Namespace).Create(secret) - if err != nil { - framework.Failf("Failed to create secrets for Ceph RBD: %v", err) - } - - return config, pod, secret, ip -} - -// CreateStorageServer is a wrapper for StartVolumeServer(). A storage server config is passed in, and a pod pointer +// CreateStorageServer is a wrapper for startVolumeServer(). A storage server config is passed in, and a pod pointer // and ip address string are returned. // Note: Expect() is called so no error is returned. func CreateStorageServer(cs clientset.Interface, config TestConfig) (pod *v1.Pod, ip string) { - pod = StartVolumeServer(cs, config) + pod = startVolumeServer(cs, config) gomega.Expect(pod).NotTo(gomega.BeNil(), "storage server pod should not be nil") ip = pod.Status.PodIP gomega.Expect(len(ip)).NotTo(gomega.BeZero(), fmt.Sprintf("pod %s's IP should not be empty", pod.Name)) @@ -283,10 +216,10 @@ func CreateStorageServer(cs clientset.Interface, config TestConfig) (pod *v1.Pod return pod, ip } -// StartVolumeServer starts a container specified by config.serverImage and exports all +// startVolumeServer starts a container specified by config.serverImage and exports all // config.serverPorts from it. The returned pod should be used to get the server // IP address and create appropriate VolumeSource. -func StartVolumeServer(client clientset.Interface, config TestConfig) *v1.Pod { +func startVolumeServer(client clientset.Interface, config TestConfig) *v1.Pod { podClient := client.CoreV1().Pods(config.Namespace) portCount := len(config.ServerPorts) @@ -400,31 +333,6 @@ func StartVolumeServer(client clientset.Interface, config TestConfig) *v1.Pod { return pod } -// CleanUpVolumeServer is a wrapper of cleanup function for volume server without secret created by specific CreateStorageServer function. -func CleanUpVolumeServer(f *framework.Framework, serverPod *v1.Pod) { - CleanUpVolumeServerWithSecret(f, serverPod, nil) -} - -// CleanUpVolumeServerWithSecret is a wrapper of cleanup function for volume server with secret created by specific CreateStorageServer function. -func CleanUpVolumeServerWithSecret(f *framework.Framework, serverPod *v1.Pod, secret *v1.Secret) { - cs := f.ClientSet - ns := f.Namespace - - if secret != nil { - framework.Logf("Deleting server secret %q...", secret.Name) - err := cs.CoreV1().Secrets(ns.Name).Delete(secret.Name, &metav1.DeleteOptions{}) - if err != nil { - framework.Logf("Delete secret failed: %v", err) - } - } - - framework.Logf("Deleting server pod %q...", serverPod.Name) - err := e2epod.DeletePodWithWait(cs, serverPod) - if err != nil { - framework.Logf("Server pod delete failed: %v", err) - } -} - // TestCleanup cleans both server and client pods. func TestCleanup(f *framework.Framework, config TestConfig) { ginkgo.By(fmt.Sprint("cleaning the environment after ", config.Prefix)) @@ -623,19 +531,6 @@ func InjectContent(f *framework.Framework, config TestConfig, fsGroup *int64, fs testVolumeContent(f, injectorPod, fsGroup, fsType, tests) } -// CreateGCEVolume creates PersistentVolumeSource for GCEVolume. -func CreateGCEVolume() (*v1.PersistentVolumeSource, string) { - diskName, err := e2epv.CreatePDWithRetry() - framework.ExpectNoError(err) - return &v1.PersistentVolumeSource{ - GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ - PDName: diskName, - FSType: "ext3", - ReadOnly: false, - }, - }, diskName -} - // GenerateScriptCmd generates the corresponding command lines to execute a command. // Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh func GenerateScriptCmd(command string) []string { @@ -697,25 +592,6 @@ func generateReadBlockCmd(fullPath string, numberOfCharacters int) []string { return commands } -// GenerateWriteandExecuteScriptFileCmd generates the corresponding command lines to write a file with the given file path -// and also execute this file. -// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh -func GenerateWriteandExecuteScriptFileCmd(content, fileName, filePath string) []string { - // for windows cluster, modify the Pod spec. - if framework.NodeOSDistroIs("windows") { - scriptName := fmt.Sprintf("%s.ps1", fileName) - fullPath := filepath.Join(filePath, scriptName) - - cmd := "echo \"" + content + "\" > " + fullPath + "; .\\" + fullPath - framework.Logf("generated pod command %s", cmd) - return []string{"powershell", "/c", cmd} - } - scriptName := fmt.Sprintf("%s.sh", fileName) - fullPath := filepath.Join(filePath, scriptName) - cmd := fmt.Sprintf("echo \"%s\" > %s; chmod u+x %s; %s;", content, fullPath, fullPath, fullPath) - return []string{"/bin/sh", "-ec", cmd} -} - // GenerateSecurityContext generates the corresponding container security context with the given inputs // If the Node OS is windows, currently we will ignore the inputs and return nil. // TODO: Will modify it after windows has its own security context diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 4a4110de265..7cd0d9be21a 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -51,6 +51,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/serviceaccount" + clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/auth" e2enode "k8s.io/kubernetes/test/e2e/framework/node" @@ -64,6 +65,11 @@ import ( imageutils "k8s.io/kubernetes/test/utils/image" ) +const ( + // Template for iSCSI IQN. + iSCSIIQNTemplate = "iqn.2003-01.io.k8s:e2e.%s" +) + // NFS type nfsDriver struct { externalProvisionerPod *v1.Pod @@ -207,7 +213,7 @@ func (n *nfsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testp } func (v *nfsVolume) DeleteVolume() { - volume.CleanUpVolumeServer(v.f, v.serverPod) + cleanUpVolumeServer(v.f, v.serverPod) } // Gluster @@ -430,7 +436,7 @@ func (i *iSCSIDriver) CreateVolume(config *testsuites.PerTestConfig, volType tes cs := f.ClientSet ns := f.Namespace - c, serverPod, serverIP, iqn := volume.NewISCSIServer(cs, ns.Name) + c, serverPod, serverIP, iqn := newISCSIServer(cs, ns.Name) config.ServerConfig = &c config.ClientNodeName = c.ClientNodeName return &iSCSIVolume{ @@ -441,8 +447,71 @@ func (i *iSCSIDriver) CreateVolume(config *testsuites.PerTestConfig, volType tes } } +// newISCSIServer is an iSCSI-specific wrapper for CreateStorageServer. +func newISCSIServer(cs clientset.Interface, namespace string) (config volume.TestConfig, pod *v1.Pod, ip, iqn string) { + // Generate cluster-wide unique IQN + iqn = fmt.Sprintf(iSCSIIQNTemplate, namespace) + config = volume.TestConfig{ + Namespace: namespace, + Prefix: "iscsi", + ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer), + ServerArgs: []string{iqn}, + ServerVolumes: map[string]string{ + // iSCSI container needs to insert modules from the host + "/lib/modules": "/lib/modules", + // iSCSI container needs to configure kernel + "/sys/kernel": "/sys/kernel", + // iSCSI source "block devices" must be available on the host + "/srv/iscsi": "/srv/iscsi", + }, + ServerReadyMessage: "iscsi target started", + ServerHostNetwork: true, + } + pod, ip = volume.CreateStorageServer(cs, config) + // Make sure the client runs on the same node as server so we don't need to open any firewalls. + config.ClientNodeName = pod.Spec.NodeName + return config, pod, ip, iqn +} + +// newRBDServer is a CephRBD-specific wrapper for CreateStorageServer. +func newRBDServer(cs clientset.Interface, namespace string) (config volume.TestConfig, pod *v1.Pod, secret *v1.Secret, ip string) { + config = volume.TestConfig{ + Namespace: namespace, + Prefix: "rbd", + ServerImage: imageutils.GetE2EImage(imageutils.VolumeRBDServer), + ServerPorts: []int{6789}, + ServerVolumes: map[string]string{ + "/lib/modules": "/lib/modules", + }, + ServerReadyMessage: "Ceph is ready", + } + pod, ip = volume.CreateStorageServer(cs, config) + // create secrets for the server + secret = &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: config.Prefix + "-secret", + }, + Data: map[string][]byte{ + // from test/images/volumes-tester/rbd/keyring + "key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="), + }, + Type: "kubernetes.io/rbd", + } + + secret, err := cs.CoreV1().Secrets(config.Namespace).Create(secret) + if err != nil { + framework.Failf("Failed to create secrets for Ceph RBD: %v", err) + } + + return config, pod, secret, ip +} + func (v *iSCSIVolume) DeleteVolume() { - volume.CleanUpVolumeServer(v.f, v.serverPod) + cleanUpVolumeServer(v.f, v.serverPod) } // Ceph RBD @@ -559,7 +628,7 @@ func (r *rbdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testp cs := f.ClientSet ns := f.Namespace - c, serverPod, secret, serverIP := volume.NewRBDServer(cs, ns.Name) + c, serverPod, secret, serverIP := newRBDServer(cs, ns.Name) config.ServerConfig = &c return &rbdVolume{ serverPod: serverPod, @@ -570,7 +639,7 @@ func (r *rbdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testp } func (v *rbdVolume) DeleteVolume() { - volume.CleanUpVolumeServerWithSecret(v.f, v.serverPod, v.secret) + cleanUpVolumeServerWithSecret(v.f, v.serverPod, v.secret) } // Ceph @@ -669,7 +738,7 @@ func (c *cephFSDriver) CreateVolume(config *testsuites.PerTestConfig, volType te cs := f.ClientSet ns := f.Namespace - cfg, serverPod, secret, serverIP := volume.NewRBDServer(cs, ns.Name) + cfg, serverPod, secret, serverIP := newRBDServer(cs, ns.Name) config.ServerConfig = &cfg return &cephVolume{ serverPod: serverPod, @@ -680,7 +749,7 @@ func (c *cephFSDriver) CreateVolume(config *testsuites.PerTestConfig, volType te } func (v *cephVolume) DeleteVolume() { - volume.CleanUpVolumeServerWithSecret(v.f, v.serverPod, v.secret) + cleanUpVolumeServerWithSecret(v.f, v.serverPod, v.secret) } // Hostpath @@ -1838,3 +1907,28 @@ func (l *localDriver) GetPersistentVolumeSource(readOnly bool, fsType string, vo }, }, l.nodeAffinityForNode(lv.ltr.Node) } + +// cleanUpVolumeServer is a wrapper of cleanup function for volume server without secret created by specific CreateStorageServer function. +func cleanUpVolumeServer(f *framework.Framework, serverPod *v1.Pod) { + cleanUpVolumeServerWithSecret(f, serverPod, nil) +} + +// cleanUpVolumeServerWithSecret is a wrapper of cleanup function for volume server with secret created by specific CreateStorageServer function. +func cleanUpVolumeServerWithSecret(f *framework.Framework, serverPod *v1.Pod, secret *v1.Secret) { + cs := f.ClientSet + ns := f.Namespace + + if secret != nil { + framework.Logf("Deleting server secret %q...", secret.Name) + err := cs.CoreV1().Secrets(ns.Name).Delete(secret.Name, &metav1.DeleteOptions{}) + if err != nil { + framework.Logf("Delete secret failed: %v", err) + } + } + + framework.Logf("Deleting server pod %q...", serverPod.Name) + err := e2epod.DeletePodWithWait(cs, serverPod) + if err != nil { + framework.Logf("Server pod delete failed: %v", err) + } +} diff --git a/test/e2e/storage/nfs_persistent_volume-disruptive.go b/test/e2e/storage/nfs_persistent_volume-disruptive.go index 0ce61d875dd..0a2fcb39492 100644 --- a/test/e2e/storage/nfs_persistent_volume-disruptive.go +++ b/test/e2e/storage/nfs_persistent_volume-disruptive.go @@ -150,7 +150,7 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() { framework.SkipUnlessSSHKeyPresent() ginkgo.By("Initializing first PD with PVPVC binding") - pvSource1, diskName1 = volume.CreateGCEVolume() + pvSource1, diskName1 = createGCEVolume() framework.ExpectNoError(err) pvConfig1 = e2epv.PersistentVolumeConfig{ NamePrefix: "gce-", @@ -163,7 +163,7 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() { framework.ExpectNoError(e2epv.WaitOnPVandPVC(c, ns, pv1, pvc1)) ginkgo.By("Initializing second PD with PVPVC binding") - pvSource2, diskName2 = volume.CreateGCEVolume() + pvSource2, diskName2 = createGCEVolume() framework.ExpectNoError(err) pvConfig2 = e2epv.PersistentVolumeConfig{ NamePrefix: "gce-", @@ -274,6 +274,19 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() { }) }) +// createGCEVolume creates PersistentVolumeSource for GCEVolume. +func createGCEVolume() (*v1.PersistentVolumeSource, string) { + diskName, err := e2epv.CreatePDWithRetry() + framework.ExpectNoError(err) + return &v1.PersistentVolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: diskName, + FSType: "ext3", + ReadOnly: false, + }, + }, diskName +} + // initTestCase initializes spec resources (pv, pvc, and pod) and returns pointers to be consumed // by the test. func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig e2epv.PersistentVolumeConfig, pvcConfig e2epv.PersistentVolumeClaimConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) { diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index 90de74d55b6..a07336e35af 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -23,6 +23,7 @@ package testsuites import ( "fmt" + "path/filepath" "github.com/onsi/ginkgo" @@ -216,7 +217,7 @@ func testScriptInPod( } else { content = fmt.Sprintf("ls %s", volPath) } - command := volume.GenerateWriteandExecuteScriptFileCmd(content, fileName, volPath) + command := generateWriteandExecuteScriptFileCmd(content, fileName, volPath) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("exec-volume-test-%s", suffix), @@ -254,3 +255,22 @@ func testScriptInPod( err := e2epod.DeletePodWithWait(f.ClientSet, pod) framework.ExpectNoError(err, "while deleting pod") } + +// generateWriteandExecuteScriptFileCmd generates the corresponding command lines to write a file with the given file path +// and also execute this file. +// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh +func generateWriteandExecuteScriptFileCmd(content, fileName, filePath string) []string { + // for windows cluster, modify the Pod spec. + if framework.NodeOSDistroIs("windows") { + scriptName := fmt.Sprintf("%s.ps1", fileName) + fullPath := filepath.Join(filePath, scriptName) + + cmd := "echo \"" + content + "\" > " + fullPath + "; .\\" + fullPath + framework.Logf("generated pod command %s", cmd) + return []string{"powershell", "/c", cmd} + } + scriptName := fmt.Sprintf("%s.sh", fileName) + fullPath := filepath.Join(filePath, scriptName) + cmd := fmt.Sprintf("echo \"%s\" > %s; chmod u+x %s; %s;", content, fullPath, fullPath, fullPath) + return []string{"/bin/sh", "-ec", cmd} +}