From 5bc823f141efde1e086f6696c26a197b78f9cf52 Mon Sep 17 00:00:00 2001 From: NickrenREN Date: Fri, 11 May 2018 19:00:10 +0800 Subject: [PATCH] fix and update local volume e2es --- test/e2e/storage/persistent_volumes-local.go | 142 +++++++++++-------- 1 file changed, 82 insertions(+), 60 deletions(-) diff --git a/test/e2e/storage/persistent_volumes-local.go b/test/e2e/storage/persistent_volumes-local.go index 8cc3b1317cc..7340f9ab527 100644 --- a/test/e2e/storage/persistent_volumes-local.go +++ b/test/e2e/storage/persistent_volumes-local.go @@ -78,8 +78,12 @@ const ( GCELocalSSDVolumeType localVolumeType = "gce-localssd-scsi-fs" // Creates a local file, formats it, and maps it as a block device. BlockLocalVolumeType localVolumeType = "block" - // Creates a local file, formats it, and mounts it to use as local volume. - BlockFsLocalVolumeType localVolumeType = "blockfs" + // Creates a local file serving as the backing for block device., formats it, + // and mounts it to use as FS mode local volume. + BlockFsWithFormatLocalVolumeType localVolumeType = "blockfswithformat" + // Creates a local file serving as the backing for block device. do not format it manually, + // and mounts it to use as FS mode local volume. + BlockFsWithoutFormatLocalVolumeType localVolumeType = "blockfswithoutformat" ) var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *localTestVolume{ @@ -90,7 +94,8 @@ var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) * DirectoryBindMountedLocalVolumeType: setupLocalVolumeDirectoryBindMounted, DirectoryLinkBindMountedLocalVolumeType: setupLocalVolumeDirectoryLinkBindMounted, BlockLocalVolumeType: setupLocalVolumeBlock, - BlockFsLocalVolumeType: setupLocalVolumeBlockFs, + BlockFsWithFormatLocalVolumeType: setupLocalVolumeBlockFsWithFormat, + BlockFsWithoutFormatLocalVolumeType: setupLocalVolumeBlockFsWithoutFormat, } var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTestVolume){ @@ -101,7 +106,8 @@ var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTes DirectoryBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryBindMounted, DirectoryLinkBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryLinkBindMounted, BlockLocalVolumeType: cleanupLocalVolumeBlock, - BlockFsLocalVolumeType: cleanupLocalVolumeBlockFs, + BlockFsWithFormatLocalVolumeType: cleanupLocalVolumeBlockFsWithFormat, + BlockFsWithoutFormatLocalVolumeType: cleanupLocalVolumeBlockFsWithoutFormat, } type localTestVolume struct { @@ -247,6 +253,11 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { pod1, pod1Err = createLocalPod(config, testVol, nil) Expect(pod1Err).NotTo(HaveOccurred()) verifyLocalPod(config, testVol, pod1, config.node0.Name) + + writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType) + + By("Writing in pod1") + podRWCmdExec(pod1, writeCmd) }) AfterEach(func() { @@ -256,16 +267,16 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { It("should be able to mount volume and read from pod1", func() { By("Reading in pod1") - // testFileContent was written during setupLocalVolume + // testFileContent was written in BeforeEach testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType) }) It("should be able to mount volume and write from pod1", func() { - // testFileContent was written during setupLocalVolume + // testFileContent was written in BeforeEach testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType) By("Writing in pod1") - writeCmd, _ := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVolType) + writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVolType) podRWCmdExec(pod1, writeCmd) }) }) @@ -346,12 +357,12 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { Context("Local volume that cannot be mounted [Slow]", func() { // TODO: - // - check for these errors in unit tests intead + // - check for these errors in unit tests instead It("should fail due to non-existent path", func() { ep := &eventPatterns{ reason: "FailedMount", pattern: make([]string, 2)} - ep.pattern = append(ep.pattern, "MountVolume.SetUp failed") + ep.pattern = append(ep.pattern, "MountVolume.NewMounter initialization failed") testVol := &localTestVolume{ node: config.node0, @@ -461,7 +472,7 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { // Delete the persistent volume claim: file will be cleaned up and volume be re-created. By("Deleting the persistent volume claim to clean up persistent volume and re-create one") - writeCmd, _ := createWriteAndReadCmds(volumePath, testFile, testFileContent, DirectoryLocalVolumeType) + writeCmd := createWriteCmd(volumePath, testFile, testFileContent, DirectoryLocalVolumeType) err = issueNodeCommand(config, writeCmd, config.node0) Expect(err).NotTo(HaveOccurred()) err = config.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{}) @@ -731,8 +742,6 @@ func testPodWithNodeConflict(config *localTestConfig, testVolType localVolumeTyp err = framework.WaitForPodNameUnschedulableInNamespace(config.client, pod.Name, pod.Namespace) Expect(err).NotTo(HaveOccurred()) - - cleanupLocalVolumes(config, []*localTestVolume{testVol}) } type eventPatterns struct { @@ -766,7 +775,12 @@ func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) { Expect(pod1Err).NotTo(HaveOccurred()) verifyLocalPod(config, testVol, pod1, config.node0.Name) - // testFileContent was written during setupLocalVolume + writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType) + + By("Writing in pod1") + podRWCmdExec(pod1, writeCmd) + + // testFileContent was written after creating pod1 testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType) By("Creating pod2 to read from the PV") @@ -774,16 +788,16 @@ func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) { Expect(pod2Err).NotTo(HaveOccurred()) verifyLocalPod(config, testVol, pod2, config.node0.Name) - // testFileContent was written during setupLocalVolume + // testFileContent was written after creating pod1 testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType) - writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType) + writeCmd = createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType) - By("Writing in pod1") - podRWCmdExec(pod1, writeCmd) + By("Writing in pod2") + podRWCmdExec(pod2, writeCmd) - By("Reading in pod2") - testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2, testVol.localVolumeType) + By("Reading in pod1") + testReadFileContent(volumeDir, testFile, testVol.hostDir, pod1, testVol.localVolumeType) By("Deleting pod1") framework.DeletePodOrFail(config.client, config.ns, pod1.Name) @@ -798,14 +812,14 @@ func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolum Expect(pod1Err).NotTo(HaveOccurred()) verifyLocalPod(config, testVol, pod1, config.node0.Name) - // testFileContent was written during setupLocalVolume - testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType) - - writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType) + writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType) By("Writing in pod1") podRWCmdExec(pod1, writeCmd) + // testFileContent was written after creating pod1 + testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType) + By("Deleting pod1") framework.DeletePodOrFail(config.client, config.ns, pod1.Name) @@ -815,7 +829,7 @@ func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolum verifyLocalPod(config, testVol, pod2, config.node0.Name) By("Reading in pod2") - testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2, testVol.localVolumeType) + testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType) By("Deleting pod2") framework.DeletePodOrFail(config.client, config.ns, pod2.Name) @@ -885,11 +899,13 @@ func cleanupLocalVolumes(config *localTestConfig, volumes []*localTestVolume) { } } -func setupWriteTestFile(hostDir string, config *localTestConfig, localVolumeType localVolumeType, node *v1.Node) *localTestVolume { - writeCmd, _ := createWriteAndReadCmds(hostDir, testFile, testFileContent, localVolumeType) - By(fmt.Sprintf("Creating test file on node %q in path %q", node.Name, hostDir)) - err := issueNodeCommand(config, writeCmd, node) - Expect(err).NotTo(HaveOccurred()) +func generateLocalTestVolume(hostDir string, config *localTestConfig, localVolumeType localVolumeType, node *v1.Node) *localTestVolume { + if localVolumeType != BlockLocalVolumeType && localVolumeType != BlockFsWithoutFormatLocalVolumeType { + mkdirCmd := fmt.Sprintf("mkdir -p %s", hostDir) + err := issueNodeCommand(config, mkdirCmd, node) + Expect(err).NotTo(HaveOccurred()) + } + return &localTestVolume{ node: node, hostDir: hostDir, @@ -901,8 +917,7 @@ func setupLocalVolumeTmpfs(config *localTestConfig, node *v1.Node) *localTestVol testDirName := "local-volume-test-" + string(uuid.NewUUID()) hostDir := filepath.Join(hostBase, testDirName) createAndMountTmpfsLocalVolume(config, hostDir, node) - // populate volume with testFile containing testFileContent - return setupWriteTestFile(hostDir, config, TmpfsLocalVolumeType, node) + return generateLocalTestVolume(hostDir, config, TmpfsLocalVolumeType, node) } func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localTestVolume { @@ -910,15 +925,13 @@ func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localT Expect(err).NotTo(HaveOccurred()) dirName := strings.Fields(res)[0] hostDir := "/mnt/disks/by-uuid/google-local-ssds-scsi-fs/" + dirName - // Populate volume with testFile containing testFileContent. - return setupWriteTestFile(hostDir, config, GCELocalSSDVolumeType, node) + return generateLocalTestVolume(hostDir, config, GCELocalSSDVolumeType, node) } func setupLocalVolumeDirectory(config *localTestConfig, node *v1.Node) *localTestVolume { testDirName := "local-volume-test-" + string(uuid.NewUUID()) hostDir := filepath.Join(hostBase, testDirName) - // Populate volume with testFile containing testFileContent. - return setupWriteTestFile(hostDir, config, DirectoryLocalVolumeType, node) + return generateLocalTestVolume(hostDir, config, DirectoryLocalVolumeType, node) } // launchNodeExecPodForLocalPV launches a hostexec pod for local PV and waits @@ -992,11 +1005,10 @@ func setupLocalVolumeDirectoryLink(config *localTestConfig, node *v1.Node) *loca testDirName := "local-volume-test-" + string(uuid.NewUUID()) hostDir := filepath.Join(hostBase, testDirName) hostDirBackend := hostDir + "-backend" - cmd := fmt.Sprintf("mkdir %s && ln -s %s %s", hostDirBackend, hostDirBackend, hostDir) + cmd := fmt.Sprintf("mkdir %s && sudo ln -s %s %s", hostDirBackend, hostDirBackend, hostDir) _, err := issueNodeCommandWithResult(config, cmd, node) Expect(err).NotTo(HaveOccurred()) - // Populate volume with testFile containing testFileContent. - return setupWriteTestFile(hostDir, config, DirectoryLinkLocalVolumeType, node) + return generateLocalTestVolume(hostDir, config, DirectoryLinkLocalVolumeType, node) } func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume { @@ -1005,20 +1017,18 @@ func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s", hostDir, hostDir, hostDir) _, err := issueNodeCommandWithResult(config, cmd, node) Expect(err).NotTo(HaveOccurred()) - // Populate volume with testFile containing testFileContent. - return setupWriteTestFile(hostDir, config, DirectoryBindMountedLocalVolumeType, node) + return generateLocalTestVolume(hostDir, config, DirectoryBindMountedLocalVolumeType, node) } func setupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume { testDirName := "local-volume-test-" + string(uuid.NewUUID()) hostDir := filepath.Join(hostBase, testDirName) hostDirBackend := hostDir + "-backend" - cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && ln -s %s %s", + cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && sudo ln -s %s %s", hostDirBackend, hostDirBackend, hostDirBackend, hostDirBackend, hostDir) _, err := issueNodeCommandWithResult(config, cmd, node) Expect(err).NotTo(HaveOccurred()) - // Populate volume with testFile containing testFileContent. - return setupWriteTestFile(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node) + return generateLocalTestVolume(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node) } func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVolume { @@ -1026,14 +1036,13 @@ func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVol hostDir := filepath.Join(hostBase, testDirName) createAndMapBlockLocalVolume(config, hostDir, node) loopDev := getBlockLoopDev(config, hostDir, node) - // Populate block volume with testFile containing testFileContent. - volume := setupWriteTestFile(loopDev, config, BlockLocalVolumeType, node) + volume := generateLocalTestVolume(loopDev, config, BlockLocalVolumeType, node) volume.hostDir = loopDev volume.loopDevDir = hostDir return volume } -func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestVolume { +func setupLocalVolumeBlockFsWithFormat(config *localTestConfig, node *v1.Node) *localTestVolume { testDirName := "local-volume-test-" + string(uuid.NewUUID()) hostDir := filepath.Join(hostBase, testDirName) createAndMapBlockLocalVolume(config, hostDir, node) @@ -1043,13 +1052,25 @@ func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestV cmd := fmt.Sprintf("sudo mkfs -t ext4 %s && sudo mount -t ext4 %s %s && sudo chmod o+rwx %s", loopDev, loopDev, hostDir, hostDir) _, err := issueNodeCommandWithResult(config, cmd, node) Expect(err).NotTo(HaveOccurred()) - // Populate block volume with testFile containing testFileContent. - volume := setupWriteTestFile(hostDir, config, BlockFsLocalVolumeType, node) + volume := generateLocalTestVolume(hostDir, config, BlockFsWithFormatLocalVolumeType, node) volume.hostDir = hostDir volume.loopDevDir = loopDev return volume } +func setupLocalVolumeBlockFsWithoutFormat(config *localTestConfig, node *v1.Node) *localTestVolume { + testDirName := "local-volume-test-" + string(uuid.NewUUID()) + hostDir := filepath.Join(hostBase, testDirName) + createAndMapBlockLocalVolume(config, hostDir, node) + loopDev := getBlockLoopDev(config, hostDir, node) + volume := generateLocalTestVolume(loopDev, config, BlockFsWithoutFormatLocalVolumeType, node) + // we do this in order to set block device path to local PV spec path directly + // and test local volume plugin FileSystem mode on block device + volume.hostDir = loopDev + volume.loopDevDir = hostDir + return volume +} + // Determine the /dev/loopXXX device associated with this test, via its hostDir. func getBlockLoopDev(config *localTestConfig, hostDir string, node *v1.Node) string { loopDevCmd := fmt.Sprintf("E2E_LOOP_DEV=$(sudo losetup | grep %s/file | awk '{ print $1 }') 2>&1 > /dev/null && echo ${E2E_LOOP_DEV}", hostDir) @@ -1100,7 +1121,7 @@ func cleanupLocalVolumeDirectoryLink(config *localTestConfig, volume *localTestV By("Removing the test directory") hostDir := volume.hostDir hostDirBackend := hostDir + "-backend" - removeCmd := fmt.Sprintf("rm -r %s && rm -r %s", hostDir, hostDirBackend) + removeCmd := fmt.Sprintf("sudo rm -r %s && rm -r %s", hostDir, hostDirBackend) err := issueNodeCommand(config, removeCmd, volume.node) Expect(err).NotTo(HaveOccurred()) } @@ -1119,7 +1140,7 @@ func cleanupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, volume By("Removing the test directory") hostDir := volume.hostDir hostDirBackend := hostDir + "-backend" - removeCmd := fmt.Sprintf("rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend) + removeCmd := fmt.Sprintf("sudo rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend) err := issueNodeCommand(config, removeCmd, volume.node) Expect(err).NotTo(HaveOccurred()) } @@ -1135,7 +1156,7 @@ func cleanupLocalVolumeBlock(config *localTestConfig, volume *localTestVolume) { } // Deletes the PVC/PV and removes the test directory holding the block file. -func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume) { +func cleanupLocalVolumeBlockFsWithFormat(config *localTestConfig, volume *localTestVolume) { // umount first By("Umount blockfs mountpoint") umountCmd := fmt.Sprintf("sudo umount %s", volume.hostDir) @@ -1147,6 +1168,15 @@ func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume) Expect(err).NotTo(HaveOccurred()) } +func cleanupLocalVolumeBlockFsWithoutFormat(config *localTestConfig, volume *localTestVolume) { + volume.hostDir = volume.loopDevDir + unmapBlockLocalVolume(config, volume.hostDir, volume.node) + By("Removing the test directory") + removeCmd := fmt.Sprintf("rm -r %s", volume.hostDir) + err := issueNodeCommand(config, removeCmd, volume.node) + Expect(err).NotTo(HaveOccurred()) +} + func makeLocalPVCConfig(config *localTestConfig, volumeType localVolumeType) framework.PersistentVolumeClaimConfig { pvcConfig := framework.PersistentVolumeClaimConfig{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, @@ -1337,14 +1367,6 @@ func unmapBlockLocalVolume(config *localTestConfig, dir string, node *v1.Node) { Expect(err).NotTo(HaveOccurred()) } -// Create corresponding write and read commands -// to be executed via hostexec Pod on the node with the local PV -func createWriteAndReadCmds(testFileDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) (writeCmd string, readCmd string) { - writeCmd = createWriteCmd(testFileDir, testFile, writeTestFileContent, volumeType) - readCmd = createReadCmd(testFileDir, testFile, volumeType) - return writeCmd, readCmd -} - func createWriteCmd(testDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) string { if volumeType == BlockLocalVolumeType { // testDir is the block device. @@ -1378,7 +1400,7 @@ func createReadCmd(testFileDir string, testFile string, volumeType localVolumeTy // Read testFile and evaluate whether it contains the testFileContent func testReadFileContent(testFileDir string, testFile string, testFileContent string, pod *v1.Pod, volumeType localVolumeType) { - readCmd := createReadCmd(volumeDir, testFile, volumeType) + readCmd := createReadCmd(testFileDir, testFile, volumeType) readOut := podRWCmdExec(pod, readCmd) Expect(readOut).To(ContainSubstring(testFileContent)) }