fix and update local volume e2es

test/e2e/storage/persistent_volumes-local.go

@@ -78,8 +78,12 @@ const (
 	GCELocalSSDVolumeType localVolumeType = "gce-localssd-scsi-fs"
 	// Creates a local file, formats it, and maps it as a block device.
 	BlockLocalVolumeType localVolumeType = "block"
-	// Creates a local file, formats it, and mounts it to use as local volume.
+	// Creates a local file serving as the backing for block device., formats it,
-	BlockFsLocalVolumeType localVolumeType = "blockfs"
+	// and mounts it to use as FS mode local volume.
+	BlockFsWithFormatLocalVolumeType localVolumeType = "blockfswithformat"
+	// Creates a local file serving as the backing for block device. do not format it manually,
+	// and mounts it to use as FS mode local volume.
+	BlockFsWithoutFormatLocalVolumeType localVolumeType = "blockfswithoutformat"
 )
 
 var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *localTestVolume{
@@ -90,7 +94,8 @@ var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *
 	DirectoryBindMountedLocalVolumeType:     setupLocalVolumeDirectoryBindMounted,
 	DirectoryLinkBindMountedLocalVolumeType: setupLocalVolumeDirectoryLinkBindMounted,
 	BlockLocalVolumeType:                    setupLocalVolumeBlock,
-	BlockFsLocalVolumeType:                  setupLocalVolumeBlockFs,
+	BlockFsWithFormatLocalVolumeType:        setupLocalVolumeBlockFsWithFormat,
+	BlockFsWithoutFormatLocalVolumeType:     setupLocalVolumeBlockFsWithoutFormat,
 }
 
 var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTestVolume){
@@ -101,7 +106,8 @@ var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTes
 	DirectoryBindMountedLocalVolumeType:     cleanupLocalVolumeDirectoryBindMounted,
 	DirectoryLinkBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryLinkBindMounted,
 	BlockLocalVolumeType:                    cleanupLocalVolumeBlock,
-	BlockFsLocalVolumeType:                  cleanupLocalVolumeBlockFs,
+	BlockFsWithFormatLocalVolumeType:        cleanupLocalVolumeBlockFsWithFormat,
+	BlockFsWithoutFormatLocalVolumeType:     cleanupLocalVolumeBlockFsWithoutFormat,
 }
 
 type localTestVolume struct {
@@ -247,6 +253,11 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 					pod1, pod1Err = createLocalPod(config, testVol, nil)
 					Expect(pod1Err).NotTo(HaveOccurred())
 					verifyLocalPod(config, testVol, pod1, config.node0.Name)
+
+					writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
+
+					By("Writing in pod1")
+					podRWCmdExec(pod1, writeCmd)
 				})
 
 				AfterEach(func() {
@@ -256,16 +267,16 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 
 				It("should be able to mount volume and read from pod1", func() {
 					By("Reading in pod1")
-					// testFileContent was written during setupLocalVolume
+					// testFileContent was written in BeforeEach
 					testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType)
 				})
 
 				It("should be able to mount volume and write from pod1", func() {
-					// testFileContent was written during setupLocalVolume
+					// testFileContent was written in BeforeEach
 					testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType)
 
 					By("Writing in pod1")
-					writeCmd, _ := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVolType)
+					writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVolType)
 					podRWCmdExec(pod1, writeCmd)
 				})
 			})
@@ -346,12 +357,12 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 
 	Context("Local volume that cannot be mounted [Slow]", func() {
 		// TODO:
-		// - check for these errors in unit tests intead
+		// - check for these errors in unit tests instead
 		It("should fail due to non-existent path", func() {
 			ep := &eventPatterns{
 				reason:  "FailedMount",
 				pattern: make([]string, 2)}
-			ep.pattern = append(ep.pattern, "MountVolume.SetUp failed")
+			ep.pattern = append(ep.pattern, "MountVolume.NewMounter initialization failed")
 
 			testVol := &localTestVolume{
 				node:            config.node0,
@@ -461,7 +472,7 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 
 			// Delete the persistent volume claim: file will be cleaned up and volume be re-created.
 			By("Deleting the persistent volume claim to clean up persistent volume and re-create one")
-			writeCmd, _ := createWriteAndReadCmds(volumePath, testFile, testFileContent, DirectoryLocalVolumeType)
+			writeCmd := createWriteCmd(volumePath, testFile, testFileContent, DirectoryLocalVolumeType)
 			err = issueNodeCommand(config, writeCmd, config.node0)
 			Expect(err).NotTo(HaveOccurred())
 			err = config.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{})
@@ -731,8 +742,6 @@ func testPodWithNodeConflict(config *localTestConfig, testVolType localVolumeTyp
 
 	err = framework.WaitForPodNameUnschedulableInNamespace(config.client, pod.Name, pod.Namespace)
 	Expect(err).NotTo(HaveOccurred())
-
-	cleanupLocalVolumes(config, []*localTestVolume{testVol})
 }
 
 type eventPatterns struct {
@@ -766,7 +775,12 @@ func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) {
 	Expect(pod1Err).NotTo(HaveOccurred())
 	verifyLocalPod(config, testVol, pod1, config.node0.Name)
 
-	// testFileContent was written during setupLocalVolume
+	writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
+
+	By("Writing in pod1")
+	podRWCmdExec(pod1, writeCmd)
+
+	// testFileContent was written after creating pod1
 	testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
 
 	By("Creating pod2 to read from the PV")
@@ -774,16 +788,16 @@ func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) {
 	Expect(pod2Err).NotTo(HaveOccurred())
 	verifyLocalPod(config, testVol, pod2, config.node0.Name)
 
-	// testFileContent was written during setupLocalVolume
+	// testFileContent was written after creating pod1
 	testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType)
 
-	writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType)
+	writeCmd = createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType)
 
-	By("Writing in pod1")
+	By("Writing in pod2")
-	podRWCmdExec(pod1, writeCmd)
+	podRWCmdExec(pod2, writeCmd)
 
-	By("Reading in pod2")
+	By("Reading in pod1")
-	testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2, testVol.localVolumeType)
+	testReadFileContent(volumeDir, testFile, testVol.hostDir, pod1, testVol.localVolumeType)
 
 	By("Deleting pod1")
 	framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
@@ -798,14 +812,14 @@ func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolum
 	Expect(pod1Err).NotTo(HaveOccurred())
 	verifyLocalPod(config, testVol, pod1, config.node0.Name)
 
-	// testFileContent was written during setupLocalVolume
+	writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
-	testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
-
-	writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType)
 
 	By("Writing in pod1")
 	podRWCmdExec(pod1, writeCmd)
 
+	// testFileContent was written after creating pod1
+	testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
+
 	By("Deleting pod1")
 	framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
 
@@ -815,7 +829,7 @@ func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolum
 	verifyLocalPod(config, testVol, pod2, config.node0.Name)
 
 	By("Reading in pod2")
-	testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2, testVol.localVolumeType)
+	testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType)
 
 	By("Deleting pod2")
 	framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
@@ -885,11 +899,13 @@ func cleanupLocalVolumes(config *localTestConfig, volumes []*localTestVolume) {
 	}
 }
 
-func setupWriteTestFile(hostDir string, config *localTestConfig, localVolumeType localVolumeType, node *v1.Node) *localTestVolume {
+func generateLocalTestVolume(hostDir string, config *localTestConfig, localVolumeType localVolumeType, node *v1.Node) *localTestVolume {
-	writeCmd, _ := createWriteAndReadCmds(hostDir, testFile, testFileContent, localVolumeType)
+	if localVolumeType != BlockLocalVolumeType && localVolumeType != BlockFsWithoutFormatLocalVolumeType {
-	By(fmt.Sprintf("Creating test file on node %q in path %q", node.Name, hostDir))
+		mkdirCmd := fmt.Sprintf("mkdir -p %s", hostDir)
-	err := issueNodeCommand(config, writeCmd, node)
+		err := issueNodeCommand(config, mkdirCmd, node)
-	Expect(err).NotTo(HaveOccurred())
+		Expect(err).NotTo(HaveOccurred())
+	}
+
 	return &localTestVolume{
 		node:            node,
 		hostDir:         hostDir,
@@ -901,8 +917,7 @@ func setupLocalVolumeTmpfs(config *localTestConfig, node *v1.Node) *localTestVol
 	testDirName := "local-volume-test-" + string(uuid.NewUUID())
 	hostDir := filepath.Join(hostBase, testDirName)
 	createAndMountTmpfsLocalVolume(config, hostDir, node)
-	// populate volume with testFile containing testFileContent
+	return generateLocalTestVolume(hostDir, config, TmpfsLocalVolumeType, node)
-	return setupWriteTestFile(hostDir, config, TmpfsLocalVolumeType, node)
 }
 
 func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localTestVolume {
@@ -910,15 +925,13 @@ func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localT
 	Expect(err).NotTo(HaveOccurred())
 	dirName := strings.Fields(res)[0]
 	hostDir := "/mnt/disks/by-uuid/google-local-ssds-scsi-fs/" + dirName
-	// Populate volume with testFile containing testFileContent.
+	return generateLocalTestVolume(hostDir, config, GCELocalSSDVolumeType, node)
-	return setupWriteTestFile(hostDir, config, GCELocalSSDVolumeType, node)
 }
 
 func setupLocalVolumeDirectory(config *localTestConfig, node *v1.Node) *localTestVolume {
 	testDirName := "local-volume-test-" + string(uuid.NewUUID())
 	hostDir := filepath.Join(hostBase, testDirName)
-	// Populate volume with testFile containing testFileContent.
+	return generateLocalTestVolume(hostDir, config, DirectoryLocalVolumeType, node)
-	return setupWriteTestFile(hostDir, config, DirectoryLocalVolumeType, node)
 }
 
 // launchNodeExecPodForLocalPV launches a hostexec pod for local PV and waits
@@ -992,11 +1005,10 @@ func setupLocalVolumeDirectoryLink(config *localTestConfig, node *v1.Node) *loca
 	testDirName := "local-volume-test-" + string(uuid.NewUUID())
 	hostDir := filepath.Join(hostBase, testDirName)
 	hostDirBackend := hostDir + "-backend"
-	cmd := fmt.Sprintf("mkdir %s && ln -s %s %s", hostDirBackend, hostDirBackend, hostDir)
+	cmd := fmt.Sprintf("mkdir %s && sudo ln -s %s %s", hostDirBackend, hostDirBackend, hostDir)
 	_, err := issueNodeCommandWithResult(config, cmd, node)
 	Expect(err).NotTo(HaveOccurred())
-	// Populate volume with testFile containing testFileContent.
+	return generateLocalTestVolume(hostDir, config, DirectoryLinkLocalVolumeType, node)
-	return setupWriteTestFile(hostDir, config, DirectoryLinkLocalVolumeType, node)
 }
 
 func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume {
@@ -1005,20 +1017,18 @@ func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node
 	cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s", hostDir, hostDir, hostDir)
 	_, err := issueNodeCommandWithResult(config, cmd, node)
 	Expect(err).NotTo(HaveOccurred())
-	// Populate volume with testFile containing testFileContent.
+	return generateLocalTestVolume(hostDir, config, DirectoryBindMountedLocalVolumeType, node)
-	return setupWriteTestFile(hostDir, config, DirectoryBindMountedLocalVolumeType, node)
 }
 
 func setupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume {
 	testDirName := "local-volume-test-" + string(uuid.NewUUID())
 	hostDir := filepath.Join(hostBase, testDirName)
 	hostDirBackend := hostDir + "-backend"
-	cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && ln -s %s %s",
+	cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && sudo ln -s %s %s",
 		hostDirBackend, hostDirBackend, hostDirBackend, hostDirBackend, hostDir)
 	_, err := issueNodeCommandWithResult(config, cmd, node)
 	Expect(err).NotTo(HaveOccurred())
-	// Populate volume with testFile containing testFileContent.
+	return generateLocalTestVolume(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node)
-	return setupWriteTestFile(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node)
 }
 
 func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVolume {
@@ -1026,14 +1036,13 @@ func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVol
 	hostDir := filepath.Join(hostBase, testDirName)
 	createAndMapBlockLocalVolume(config, hostDir, node)
 	loopDev := getBlockLoopDev(config, hostDir, node)
-	// Populate block volume with testFile containing testFileContent.
+	volume := generateLocalTestVolume(loopDev, config, BlockLocalVolumeType, node)
-	volume := setupWriteTestFile(loopDev, config, BlockLocalVolumeType, node)
 	volume.hostDir = loopDev
 	volume.loopDevDir = hostDir
 	return volume
 }
 
-func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestVolume {
+func setupLocalVolumeBlockFsWithFormat(config *localTestConfig, node *v1.Node) *localTestVolume {
 	testDirName := "local-volume-test-" + string(uuid.NewUUID())
 	hostDir := filepath.Join(hostBase, testDirName)
 	createAndMapBlockLocalVolume(config, hostDir, node)
@@ -1043,13 +1052,25 @@ func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestV
 	cmd := fmt.Sprintf("sudo mkfs -t ext4 %s && sudo mount -t ext4 %s %s && sudo chmod o+rwx %s", loopDev, loopDev, hostDir, hostDir)
 	_, err := issueNodeCommandWithResult(config, cmd, node)
 	Expect(err).NotTo(HaveOccurred())
-	// Populate block volume with testFile containing testFileContent.
+	volume := generateLocalTestVolume(hostDir, config, BlockFsWithFormatLocalVolumeType, node)
-	volume := setupWriteTestFile(hostDir, config, BlockFsLocalVolumeType, node)
 	volume.hostDir = hostDir
 	volume.loopDevDir = loopDev
 	return volume
 }
 
+func setupLocalVolumeBlockFsWithoutFormat(config *localTestConfig, node *v1.Node) *localTestVolume {
+	testDirName := "local-volume-test-" + string(uuid.NewUUID())
+	hostDir := filepath.Join(hostBase, testDirName)
+	createAndMapBlockLocalVolume(config, hostDir, node)
+	loopDev := getBlockLoopDev(config, hostDir, node)
+	volume := generateLocalTestVolume(loopDev, config, BlockFsWithoutFormatLocalVolumeType, node)
+	// we do this in order to set block device path to local PV spec path directly
+	// and test local volume plugin FileSystem mode on block device
+	volume.hostDir = loopDev
+	volume.loopDevDir = hostDir
+	return volume
+}
+
 // Determine the /dev/loopXXX device associated with this test, via its hostDir.
 func getBlockLoopDev(config *localTestConfig, hostDir string, node *v1.Node) string {
 	loopDevCmd := fmt.Sprintf("E2E_LOOP_DEV=$(sudo losetup | grep %s/file | awk '{ print $1 }') 2>&1 > /dev/null && echo ${E2E_LOOP_DEV}", hostDir)
@@ -1100,7 +1121,7 @@ func cleanupLocalVolumeDirectoryLink(config *localTestConfig, volume *localTestV
 	By("Removing the test directory")
 	hostDir := volume.hostDir
 	hostDirBackend := hostDir + "-backend"
-	removeCmd := fmt.Sprintf("rm -r %s && rm -r %s", hostDir, hostDirBackend)
+	removeCmd := fmt.Sprintf("sudo rm -r %s && rm -r %s", hostDir, hostDirBackend)
 	err := issueNodeCommand(config, removeCmd, volume.node)
 	Expect(err).NotTo(HaveOccurred())
 }
@@ -1119,7 +1140,7 @@ func cleanupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, volume
 	By("Removing the test directory")
 	hostDir := volume.hostDir
 	hostDirBackend := hostDir + "-backend"
-	removeCmd := fmt.Sprintf("rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend)
+	removeCmd := fmt.Sprintf("sudo rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend)
 	err := issueNodeCommand(config, removeCmd, volume.node)
 	Expect(err).NotTo(HaveOccurred())
 }
@@ -1135,7 +1156,7 @@ func cleanupLocalVolumeBlock(config *localTestConfig, volume *localTestVolume) {
 }
 
 // Deletes the PVC/PV and removes the test directory holding the block file.
-func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume) {
+func cleanupLocalVolumeBlockFsWithFormat(config *localTestConfig, volume *localTestVolume) {
 	// umount first
 	By("Umount blockfs mountpoint")
 	umountCmd := fmt.Sprintf("sudo umount %s", volume.hostDir)
@@ -1147,6 +1168,15 @@ func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume)
 	Expect(err).NotTo(HaveOccurred())
 }
 
+func cleanupLocalVolumeBlockFsWithoutFormat(config *localTestConfig, volume *localTestVolume) {
+	volume.hostDir = volume.loopDevDir
+	unmapBlockLocalVolume(config, volume.hostDir, volume.node)
+	By("Removing the test directory")
+	removeCmd := fmt.Sprintf("rm -r %s", volume.hostDir)
+	err := issueNodeCommand(config, removeCmd, volume.node)
+	Expect(err).NotTo(HaveOccurred())
+}
+
 func makeLocalPVCConfig(config *localTestConfig, volumeType localVolumeType) framework.PersistentVolumeClaimConfig {
 	pvcConfig := framework.PersistentVolumeClaimConfig{
 		AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
@@ -1337,14 +1367,6 @@ func unmapBlockLocalVolume(config *localTestConfig, dir string, node *v1.Node) {
 	Expect(err).NotTo(HaveOccurred())
 }
 
-// Create corresponding write and read commands
-// to be executed via hostexec Pod on the node with the local PV
-func createWriteAndReadCmds(testFileDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) (writeCmd string, readCmd string) {
-	writeCmd = createWriteCmd(testFileDir, testFile, writeTestFileContent, volumeType)
-	readCmd = createReadCmd(testFileDir, testFile, volumeType)
-	return writeCmd, readCmd
-}
-
 func createWriteCmd(testDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) string {
 	if volumeType == BlockLocalVolumeType {
 		// testDir is the block device.
@@ -1378,7 +1400,7 @@ func createReadCmd(testFileDir string, testFile string, volumeType localVolumeTy
 
 // Read testFile and evaluate whether it contains the testFileContent
 func testReadFileContent(testFileDir string, testFile string, testFileContent string, pod *v1.Pod, volumeType localVolumeType) {
-	readCmd := createReadCmd(volumeDir, testFile, volumeType)
+	readCmd := createReadCmd(testFileDir, testFile, volumeType)
 	readOut := podRWCmdExec(pod, readCmd)
 	Expect(readOut).To(ContainSubstring(testFileContent))
 }