mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-26 21:17:23 +00:00
Merge pull request #94881 from nixpanic/test/e2e/storage/dd-without-cache
e2e: skip write/read cache when running Block-mode RWX PVC tests
This commit is contained in:
commit
75448b3205
@ -48,6 +48,7 @@ type Config struct {
|
|||||||
SeLinuxLabel *v1.SELinuxOptions
|
SeLinuxLabel *v1.SELinuxOptions
|
||||||
FsGroup *int64
|
FsGroup *int64
|
||||||
NodeSelection NodeSelection
|
NodeSelection NodeSelection
|
||||||
|
ImageID int
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateUnschedulablePod with given claims based on node selector
|
// CreateUnschedulablePod with given claims based on node selector
|
||||||
@ -185,6 +186,10 @@ func MakeSecPod(podConfig *Config) (*v1.Pod, error) {
|
|||||||
return &i
|
return &i
|
||||||
}(1000)
|
}(1000)
|
||||||
}
|
}
|
||||||
|
image := imageutils.BusyBox
|
||||||
|
if podConfig.ImageID != imageutils.None {
|
||||||
|
image = podConfig.ImageID
|
||||||
|
}
|
||||||
podSpec := &v1.Pod{
|
podSpec := &v1.Pod{
|
||||||
TypeMeta: metav1.TypeMeta{
|
TypeMeta: metav1.TypeMeta{
|
||||||
Kind: "Pod",
|
Kind: "Pod",
|
||||||
@ -203,7 +208,7 @@ func MakeSecPod(podConfig *Config) (*v1.Pod, error) {
|
|||||||
Containers: []v1.Container{
|
Containers: []v1.Container{
|
||||||
{
|
{
|
||||||
Name: "write-pod",
|
Name: "write-pod",
|
||||||
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
Image: imageutils.GetE2EImage(image),
|
||||||
Command: []string{"/bin/sh"},
|
Command: []string{"/bin/sh"},
|
||||||
Args: []string{"-c", podConfig.Command},
|
Args: []string{"-c", podConfig.Command},
|
||||||
SecurityContext: &v1.SecurityContext{
|
SecurityContext: &v1.SecurityContext{
|
||||||
|
@ -35,6 +35,7 @@ import (
|
|||||||
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
|
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
|
||||||
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
|
||||||
"k8s.io/kubernetes/test/e2e/storage/utils"
|
"k8s.io/kubernetes/test/e2e/storage/utils"
|
||||||
|
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||||
)
|
)
|
||||||
|
|
||||||
type multiVolumeTestSuite struct {
|
type multiVolumeTestSuite struct {
|
||||||
@ -423,14 +424,14 @@ func testAccessMultipleVolumes(f *framework.Framework, cs clientset.Interface, n
|
|||||||
|
|
||||||
if readSeedBase > 0 {
|
if readSeedBase > 0 {
|
||||||
ginkgo.By(fmt.Sprintf("Checking if read from the volume%d works properly", index))
|
ginkgo.By(fmt.Sprintf("Checking if read from the volume%d works properly", index))
|
||||||
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, readSeedBase+int64(i))
|
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, false, path, byteLen, readSeedBase+int64(i))
|
||||||
}
|
}
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Checking if write to the volume%d works properly", index))
|
ginkgo.By(fmt.Sprintf("Checking if write to the volume%d works properly", index))
|
||||||
utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, writeSeedBase+int64(i))
|
utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, false, path, byteLen, writeSeedBase+int64(i))
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Checking if read from the volume%d works properly", index))
|
ginkgo.By(fmt.Sprintf("Checking if read from the volume%d works properly", index))
|
||||||
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, writeSeedBase+int64(i))
|
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, false, path, byteLen, writeSeedBase+int64(i))
|
||||||
}
|
}
|
||||||
|
|
||||||
pod, err = cs.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
|
pod, err = cs.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
|
||||||
@ -481,6 +482,7 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int
|
|||||||
ginkgo.By(fmt.Sprintf("Creating pod%d with a volume on %+v", index, node))
|
ginkgo.By(fmt.Sprintf("Creating pod%d with a volume on %+v", index, node))
|
||||||
podConfig := e2epod.Config{
|
podConfig := e2epod.Config{
|
||||||
NS: ns,
|
NS: ns,
|
||||||
|
ImageID: imageutils.DebianIptables,
|
||||||
PVCs: []*v1.PersistentVolumeClaim{pvc},
|
PVCs: []*v1.PersistentVolumeClaim{pvc},
|
||||||
SeLinuxLabel: e2epv.SELinuxLabel,
|
SeLinuxLabel: e2epv.SELinuxLabel,
|
||||||
NodeSelection: node,
|
NodeSelection: node,
|
||||||
@ -506,6 +508,14 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int
|
|||||||
|
|
||||||
var seed int64
|
var seed int64
|
||||||
byteLen := 64
|
byteLen := 64
|
||||||
|
directIO := false
|
||||||
|
// direct IO is needed for Block-mode PVs
|
||||||
|
if *pvc.Spec.VolumeMode == v1.PersistentVolumeBlock {
|
||||||
|
// byteLen should be the size of a sector to enable direct I/O
|
||||||
|
byteLen = 512
|
||||||
|
directIO = true
|
||||||
|
}
|
||||||
|
|
||||||
path := "/mnt/volume1"
|
path := "/mnt/volume1"
|
||||||
// Check if volume can be accessed from each pod
|
// Check if volume can be accessed from each pod
|
||||||
for i, pod := range pods {
|
for i, pod := range pods {
|
||||||
@ -521,17 +531,17 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int
|
|||||||
if i != 0 {
|
if i != 0 {
|
||||||
ginkgo.By(fmt.Sprintf("From pod%d, checking if reading the data that pod%d write works properly", index, index-1))
|
ginkgo.By(fmt.Sprintf("From pod%d, checking if reading the data that pod%d write works properly", index, index-1))
|
||||||
// For 1st pod, no one has written data yet, so pass the read check
|
// For 1st pod, no one has written data yet, so pass the read check
|
||||||
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
|
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update the seed and check if write/read works properly
|
// Update the seed and check if write/read works properly
|
||||||
seed = time.Now().UTC().UnixNano()
|
seed = time.Now().UTC().UnixNano()
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Checking if write to the volume in pod%d works properly", index))
|
ginkgo.By(fmt.Sprintf("Checking if write to the volume in pod%d works properly", index))
|
||||||
utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
|
utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed)
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Checking if read from the volume in pod%d works properly", index))
|
ginkgo.By(fmt.Sprintf("Checking if read from the volume in pod%d works properly", index))
|
||||||
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
|
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete the last pod and remove from slice of pods
|
// Delete the last pod and remove from slice of pods
|
||||||
@ -560,16 +570,16 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int
|
|||||||
} else {
|
} else {
|
||||||
ginkgo.By(fmt.Sprintf("From pod%d, rechecking if reading the data that pod%d write works properly", index, index-1))
|
ginkgo.By(fmt.Sprintf("From pod%d, rechecking if reading the data that pod%d write works properly", index, index-1))
|
||||||
}
|
}
|
||||||
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
|
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed)
|
||||||
|
|
||||||
// Update the seed and check if write/read works properly
|
// Update the seed and check if write/read works properly
|
||||||
seed = time.Now().UTC().UnixNano()
|
seed = time.Now().UTC().UnixNano()
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Rechecking if write to the volume in pod%d works properly", index))
|
ginkgo.By(fmt.Sprintf("Rechecking if write to the volume in pod%d works properly", index))
|
||||||
utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
|
utils.CheckWriteToPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed)
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Rechecking if read from the volume in pod%d works properly", index))
|
ginkgo.By(fmt.Sprintf("Rechecking if read from the volume in pod%d works properly", index))
|
||||||
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
|
utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, directIO, path, byteLen, seed)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -241,13 +241,13 @@ func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Fra
|
|||||||
seed := time.Now().UTC().UnixNano()
|
seed := time.Now().UTC().UnixNano()
|
||||||
|
|
||||||
ginkgo.By("Writing to the volume.")
|
ginkgo.By("Writing to the volume.")
|
||||||
CheckWriteToPath(f, clientPod, v1.PersistentVolumeFilesystem, path, byteLen, seed)
|
CheckWriteToPath(f, clientPod, v1.PersistentVolumeFilesystem, false, path, byteLen, seed)
|
||||||
|
|
||||||
ginkgo.By("Restarting kubelet")
|
ginkgo.By("Restarting kubelet")
|
||||||
KubeletCommand(KRestart, c, clientPod)
|
KubeletCommand(KRestart, c, clientPod)
|
||||||
|
|
||||||
ginkgo.By("Testing that written file is accessible.")
|
ginkgo.By("Testing that written file is accessible.")
|
||||||
CheckReadFromPath(f, clientPod, v1.PersistentVolumeFilesystem, path, byteLen, seed)
|
CheckReadFromPath(f, clientPod, v1.PersistentVolumeFilesystem, false, path, byteLen, seed)
|
||||||
|
|
||||||
framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, path)
|
framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, path)
|
||||||
}
|
}
|
||||||
@ -259,13 +259,13 @@ func TestKubeletRestartsAndRestoresMap(c clientset.Interface, f *framework.Frame
|
|||||||
seed := time.Now().UTC().UnixNano()
|
seed := time.Now().UTC().UnixNano()
|
||||||
|
|
||||||
ginkgo.By("Writing to the volume.")
|
ginkgo.By("Writing to the volume.")
|
||||||
CheckWriteToPath(f, clientPod, v1.PersistentVolumeBlock, path, byteLen, seed)
|
CheckWriteToPath(f, clientPod, v1.PersistentVolumeBlock, false, path, byteLen, seed)
|
||||||
|
|
||||||
ginkgo.By("Restarting kubelet")
|
ginkgo.By("Restarting kubelet")
|
||||||
KubeletCommand(KRestart, c, clientPod)
|
KubeletCommand(KRestart, c, clientPod)
|
||||||
|
|
||||||
ginkgo.By("Testing that written pv is accessible.")
|
ginkgo.By("Testing that written pv is accessible.")
|
||||||
CheckReadFromPath(f, clientPod, v1.PersistentVolumeBlock, path, byteLen, seed)
|
CheckReadFromPath(f, clientPod, v1.PersistentVolumeBlock, false, path, byteLen, seed)
|
||||||
|
|
||||||
framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, path)
|
framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, path)
|
||||||
}
|
}
|
||||||
@ -656,33 +656,54 @@ func genBinDataFromSeed(len int, seed int64) []byte {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CheckReadFromPath validate that file can be properly read.
|
// CheckReadFromPath validate that file can be properly read.
|
||||||
func CheckReadFromPath(f *framework.Framework, pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) {
|
//
|
||||||
|
// Note: directIO does not work with (default) BusyBox Pods. A requirement for
|
||||||
|
// directIO to function correctly, is to read whole sector(s) for Block-mode
|
||||||
|
// PVCs (normally a sector is 512 bytes), or memory pages for files (commonly
|
||||||
|
// 4096 bytes).
|
||||||
|
func CheckReadFromPath(f *framework.Framework, pod *v1.Pod, volMode v1.PersistentVolumeMode, directIO bool, path string, len int, seed int64) {
|
||||||
var pathForVolMode string
|
var pathForVolMode string
|
||||||
|
var iflag string
|
||||||
|
|
||||||
if volMode == v1.PersistentVolumeBlock {
|
if volMode == v1.PersistentVolumeBlock {
|
||||||
pathForVolMode = path
|
pathForVolMode = path
|
||||||
} else {
|
} else {
|
||||||
pathForVolMode = filepath.Join(path, "file1.txt")
|
pathForVolMode = filepath.Join(path, "file1.txt")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if directIO {
|
||||||
|
iflag = "iflag=direct"
|
||||||
|
}
|
||||||
|
|
||||||
sum := sha256.Sum256(genBinDataFromSeed(len, seed))
|
sum := sha256.Sum256(genBinDataFromSeed(len, seed))
|
||||||
|
|
||||||
VerifyExecInPodSucceed(f, pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum", pathForVolMode, len))
|
VerifyExecInPodSucceed(f, pod, fmt.Sprintf("dd if=%s %s bs=%d count=1 | sha256sum", pathForVolMode, iflag, len))
|
||||||
VerifyExecInPodSucceed(f, pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum | grep -Fq %x", pathForVolMode, len, sum))
|
VerifyExecInPodSucceed(f, pod, fmt.Sprintf("dd if=%s %s bs=%d count=1 | sha256sum | grep -Fq %x", pathForVolMode, iflag, len, sum))
|
||||||
}
|
}
|
||||||
|
|
||||||
// CheckWriteToPath that file can be properly written.
|
// CheckWriteToPath that file can be properly written.
|
||||||
func CheckWriteToPath(f *framework.Framework, pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) {
|
//
|
||||||
|
// Note: nocache does not work with (default) BusyBox Pods. To read without
|
||||||
|
// caching, enable directIO with CheckReadFromPath and check the hints about
|
||||||
|
// the len requirements.
|
||||||
|
func CheckWriteToPath(f *framework.Framework, pod *v1.Pod, volMode v1.PersistentVolumeMode, nocache bool, path string, len int, seed int64) {
|
||||||
var pathForVolMode string
|
var pathForVolMode string
|
||||||
|
var oflag string
|
||||||
|
|
||||||
if volMode == v1.PersistentVolumeBlock {
|
if volMode == v1.PersistentVolumeBlock {
|
||||||
pathForVolMode = path
|
pathForVolMode = path
|
||||||
} else {
|
} else {
|
||||||
pathForVolMode = filepath.Join(path, "file1.txt")
|
pathForVolMode = filepath.Join(path, "file1.txt")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if nocache {
|
||||||
|
oflag = "oflag=nocache"
|
||||||
|
}
|
||||||
|
|
||||||
encoded := base64.StdEncoding.EncodeToString(genBinDataFromSeed(len, seed))
|
encoded := base64.StdEncoding.EncodeToString(genBinDataFromSeed(len, seed))
|
||||||
|
|
||||||
VerifyExecInPodSucceed(f, pod, fmt.Sprintf("echo %s | base64 -d | sha256sum", encoded))
|
VerifyExecInPodSucceed(f, pod, fmt.Sprintf("echo %s | base64 -d | sha256sum", encoded))
|
||||||
VerifyExecInPodSucceed(f, pod, fmt.Sprintf("echo %s | base64 -d | dd of=%s bs=%d count=1", encoded, pathForVolMode, len))
|
VerifyExecInPodSucceed(f, pod, fmt.Sprintf("echo %s | base64 -d | dd of=%s %s bs=%d count=1", encoded, pathForVolMode, oflag, len))
|
||||||
}
|
}
|
||||||
|
|
||||||
// findMountPoints returns all mount points on given node under specified directory.
|
// findMountPoints returns all mount points on given node under specified directory.
|
||||||
|
@ -119,8 +119,10 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
// None is to be used for unset/default images
|
||||||
|
None = iota
|
||||||
// Agnhost image
|
// Agnhost image
|
||||||
Agnhost = iota
|
Agnhost
|
||||||
// AgnhostPrivate image
|
// AgnhostPrivate image
|
||||||
AgnhostPrivate
|
AgnhostPrivate
|
||||||
// APIServer image
|
// APIServer image
|
||||||
|
Loading…
Reference in New Issue
Block a user