diff --git a/test/e2e/storage/testsuites/multivolume.go b/test/e2e/storage/testsuites/multivolume.go index 6934bd2a400..e4cc1640ad2 100644 --- a/test/e2e/storage/testsuites/multivolume.go +++ b/test/e2e/storage/testsuites/multivolume.go @@ -35,6 +35,7 @@ import ( e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/utils" + imageutils "k8s.io/kubernetes/test/utils/image" ) type multiVolumeTestSuite struct { @@ -423,14 +424,14 @@ func testAccessMultipleVolumes(f *framework.Framework, cs clientset.Interface, n if readSeedBase > 0 { ginkgo.By(fmt.Sprintf("Checking if read from the volume%d works properly", index)) - utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, readSeedBase+int64(i)) + utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, false, path, byteLen, readSeedBase+int64(i)) } ginkgo.By(fmt.Sprintf("Checking if write to the volume%d works properly", index)) - utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, writeSeedBase+int64(i)) + utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, false, path, byteLen, writeSeedBase+int64(i)) ginkgo.By(fmt.Sprintf("Checking if read from the volume%d works properly", index)) - utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, writeSeedBase+int64(i)) + utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, false, path, byteLen, writeSeedBase+int64(i)) } pod, err = cs.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{}) @@ -481,6 +482,7 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int ginkgo.By(fmt.Sprintf("Creating pod%d with a volume on %+v", index, node)) podConfig := e2epod.Config{ NS: ns, + ImageID: imageutils.DebianIptables, PVCs: []*v1.PersistentVolumeClaim{pvc}, SeLinuxLabel: e2epv.SELinuxLabel, NodeSelection: node, @@ -506,6 +508,14 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int var seed int64 byteLen := 64 + directIO := false + // direct IO is needed for Block-mode PVs + if *pvc.Spec.VolumeMode == v1.PersistentVolumeBlock { + // byteLen should be the size of a sector to enable direct I/O + byteLen = 512 + directIO = true + } + path := "/mnt/volume1" // Check if volume can be accessed from each pod for i, pod := range pods { @@ -521,17 +531,17 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int if i != 0 { ginkgo.By(fmt.Sprintf("From pod%d, checking if reading the data that pod%d write works properly", index, index-1)) // For 1st pod, no one has written data yet, so pass the read check - utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed) + utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed) } // Update the seed and check if write/read works properly seed = time.Now().UTC().UnixNano() ginkgo.By(fmt.Sprintf("Checking if write to the volume in pod%d works properly", index)) - utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed) + utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed) ginkgo.By(fmt.Sprintf("Checking if read from the volume in pod%d works properly", index)) - utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed) + utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed) } // Delete the last pod and remove from slice of pods @@ -560,16 +570,16 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int } else { ginkgo.By(fmt.Sprintf("From pod%d, rechecking if reading the data that pod%d write works properly", index, index-1)) } - utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed) + utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed) // Update the seed and check if write/read works properly seed = time.Now().UTC().UnixNano() ginkgo.By(fmt.Sprintf("Rechecking if write to the volume in pod%d works properly", index)) - utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed) + utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed) ginkgo.By(fmt.Sprintf("Rechecking if read from the volume in pod%d works properly", index)) - utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed) + utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed) } } diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index 7ee13f4f9ac..b046db76e1f 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -241,13 +241,13 @@ func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Fra seed := time.Now().UTC().UnixNano() ginkgo.By("Writing to the volume.") - CheckWriteToPath(f, clientPod, v1.PersistentVolumeFilesystem, path, byteLen, seed) + CheckWriteToPath(f, clientPod, v1.PersistentVolumeFilesystem, false, path, byteLen, seed) ginkgo.By("Restarting kubelet") KubeletCommand(KRestart, c, clientPod) ginkgo.By("Testing that written file is accessible.") - CheckReadFromPath(f, clientPod, v1.PersistentVolumeFilesystem, path, byteLen, seed) + CheckReadFromPath(f, clientPod, v1.PersistentVolumeFilesystem, false, path, byteLen, seed) framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, path) } @@ -259,13 +259,13 @@ func TestKubeletRestartsAndRestoresMap(c clientset.Interface, f *framework.Frame seed := time.Now().UTC().UnixNano() ginkgo.By("Writing to the volume.") - CheckWriteToPath(f, clientPod, v1.PersistentVolumeBlock, path, byteLen, seed) + CheckWriteToPath(f, clientPod, v1.PersistentVolumeBlock, false, path, byteLen, seed) ginkgo.By("Restarting kubelet") KubeletCommand(KRestart, c, clientPod) ginkgo.By("Testing that written pv is accessible.") - CheckReadFromPath(f, clientPod, v1.PersistentVolumeBlock, path, byteLen, seed) + CheckReadFromPath(f, clientPod, v1.PersistentVolumeBlock, false, path, byteLen, seed) framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, path) } @@ -656,33 +656,54 @@ func genBinDataFromSeed(len int, seed int64) []byte { } // CheckReadFromPath validate that file can be properly read. -func CheckReadFromPath(f *framework.Framework, pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) { +// +// Note: directIO does not work with (default) BusyBox Pods. A requirement for +// directIO to function correctly, is to read whole sector(s) for Block-mode +// PVCs (normally a sector is 512 bytes), or memory pages for files (commonly +// 4096 bytes). +func CheckReadFromPath(f *framework.Framework, pod *v1.Pod, volMode v1.PersistentVolumeMode, directIO bool, path string, len int, seed int64) { var pathForVolMode string + var iflag string + if volMode == v1.PersistentVolumeBlock { pathForVolMode = path } else { pathForVolMode = filepath.Join(path, "file1.txt") } + if directIO { + iflag = "iflag=direct" + } + sum := sha256.Sum256(genBinDataFromSeed(len, seed)) - VerifyExecInPodSucceed(f, pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum", pathForVolMode, len)) - VerifyExecInPodSucceed(f, pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum | grep -Fq %x", pathForVolMode, len, sum)) + VerifyExecInPodSucceed(f, pod, fmt.Sprintf("dd if=%s %s bs=%d count=1 | sha256sum", pathForVolMode, iflag, len)) + VerifyExecInPodSucceed(f, pod, fmt.Sprintf("dd if=%s %s bs=%d count=1 | sha256sum | grep -Fq %x", pathForVolMode, iflag, len, sum)) } // CheckWriteToPath that file can be properly written. -func CheckWriteToPath(f *framework.Framework, pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) { +// +// Note: nocache does not work with (default) BusyBox Pods. To read without +// caching, enable directIO with CheckReadFromPath and check the hints about +// the len requirements. +func CheckWriteToPath(f *framework.Framework, pod *v1.Pod, volMode v1.PersistentVolumeMode, nocache bool, path string, len int, seed int64) { var pathForVolMode string + var oflag string + if volMode == v1.PersistentVolumeBlock { pathForVolMode = path } else { pathForVolMode = filepath.Join(path, "file1.txt") } + if nocache { + oflag = "oflag=nocache" + } + encoded := base64.StdEncoding.EncodeToString(genBinDataFromSeed(len, seed)) VerifyExecInPodSucceed(f, pod, fmt.Sprintf("echo %s | base64 -d | sha256sum", encoded)) - VerifyExecInPodSucceed(f, pod, fmt.Sprintf("echo %s | base64 -d | dd of=%s bs=%d count=1", encoded, pathForVolMode, len)) + VerifyExecInPodSucceed(f, pod, fmt.Sprintf("echo %s | base64 -d | dd of=%s %s bs=%d count=1", encoded, pathForVolMode, oflag, len)) } // findMountPoints returns all mount points on given node under specified directory.