From 56373aa10d2f5d3f92275d0902400d8f8653bc8a Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Sun, 8 Nov 2020 23:12:26 -0800 Subject: [PATCH] Update storage test suits for Windows Update storage test suits to enable some tests on Windows Change-Id: I1f596bb9f7a0f41ae398aec43406ee5c753cdce3 --- test/e2e/framework/volume/fixtures.go | 42 ++++++++++++++++++-- test/e2e/storage/testsuites/disruptive.go | 3 +- test/e2e/storage/testsuites/ephemeral.go | 2 +- test/e2e/storage/testsuites/multivolume.go | 11 ++--- test/e2e/storage/testsuites/provisioning.go | 2 +- test/e2e/storage/testsuites/subpath.go | 14 +++---- test/e2e/storage/testsuites/volume_expand.go | 23 +++-------- test/e2e/storage/testsuites/volume_io.go | 6 +-- test/e2e/storage/testsuites/volumemode.go | 9 +++-- test/e2e/storage/testsuites/volumes.go | 2 +- test/e2e/storage/utils/utils.go | 18 +++++++-- 11 files changed, 86 insertions(+), 46 deletions(-) diff --git a/test/e2e/framework/volume/fixtures.go b/test/e2e/framework/volume/fixtures.go index e01f8b257b6..cb27ac9874b 100644 --- a/test/e2e/framework/volume/fixtures.go +++ b/test/e2e/framework/volume/fixtures.go @@ -388,7 +388,7 @@ func runVolumeTesterPod(client clientset.Interface, config TestConfig, podSuffix Containers: []v1.Container{ { Name: config.Prefix + "-" + podSuffix, - Image: GetTestImage(framework.BusyBoxImage), + Image: GetDefaultTestImage(), WorkingDir: "/opt", // An imperative and easily debuggable container which reads/writes vol contents for // us to scan in the tests or by eye. @@ -654,9 +654,45 @@ func GeneratePodSecurityContext(fsGroup *int64, seLinuxOptions *v1.SELinuxOption // GetTestImage returns the image name with the given input // If the Node OS is windows, currently we return Agnhost image for Windows node // due to the issue of #https://github.com/kubernetes-sigs/windows-testing/pull/35. -func GetTestImage(image string) string { +func GetTestImage(id int) string { if framework.NodeOSDistroIs("windows") { return imageutils.GetE2EImage(imageutils.Agnhost) } - return image + return imageutils.GetE2EImage(id) +} + +// GetTestImageID returns the image id with the given input +// If the Node OS is windows, currently we return Agnhost image for Windows node +// due to the issue of #https://github.com/kubernetes-sigs/windows-testing/pull/35. +func GetTestImageID(id int) int { + if framework.NodeOSDistroIs("windows") { + return imageutils.Agnhost + } + return id +} + +// GetDefaultTestImage returns the default test image based on OS. +// If the node OS is windows, currently we return Agnhost image for Windows node +// due to the issue of #https://github.com/kubernetes-sigs/windows-testing/pull/35. +// If the node OS is linux, return busybox image +func GetDefaultTestImage() string { + return imageutils.GetE2EImage(GetDefaultTestImageID()) +} + +// GetDefaultTestImageID returns the default test image id based on OS. +// If the node OS is windows, currently we return Agnhost image for Windows node +// due to the issue of #https://github.com/kubernetes-sigs/windows-testing/pull/35. +// If the node OS is linux, return busybox image +func GetDefaultTestImageID() int { + return GetTestImageID(imageutils.BusyBox) +} + +// GetLinuxLabel returns the default SELinuxLabel based on OS. +// If the node OS is windows, it will return nil +func GetLinuxLabel() *v1.SELinuxOptions { + if framework.NodeOSDistroIs("windows") { + return nil + } + return &v1.SELinuxOptions{ + Level: "s0:c0,c1"} } diff --git a/test/e2e/storage/testsuites/disruptive.go b/test/e2e/storage/testsuites/disruptive.go index 0649cb4dadb..6350752ea9d 100644 --- a/test/e2e/storage/testsuites/disruptive.go +++ b/test/e2e/storage/testsuites/disruptive.go @@ -26,6 +26,7 @@ import ( e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" "k8s.io/kubernetes/test/e2e/storage/testpatterns" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -166,7 +167,7 @@ func (s *disruptiveTestSuite) DefineTests(driver TestDriver, pattern testpattern InlineVolumeSources: inlineSources, SeLinuxLabel: e2epv.SELinuxLabel, NodeSelection: l.config.ClientNodeSelection, - ImageID: getTestImage(), + ImageID: e2evolume.GetDefaultTestImageID(), } l.pod, err = e2epod.CreateSecPodWithNodeSelection(l.cs, &podConfig, framework.PodStartTimeout) framework.ExpectNoError(err, "While creating pods for kubelet restart test") diff --git a/test/e2e/storage/testsuites/ephemeral.go b/test/e2e/storage/testsuites/ephemeral.go index 9c4f7a0e85b..cfa45ce81cc 100644 --- a/test/e2e/storage/testsuites/ephemeral.go +++ b/test/e2e/storage/testsuites/ephemeral.go @@ -352,7 +352,7 @@ func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command stri Containers: []v1.Container{ { Name: "csi-volume-tester", - Image: e2evolume.GetTestImage(framework.BusyBoxImage), + Image: e2evolume.GetDefaultTestImage(), Command: e2evolume.GenerateScriptCmd(command), }, }, diff --git a/test/e2e/storage/testsuites/multivolume.go b/test/e2e/storage/testsuites/multivolume.go index e4cc1640ad2..9c225b515b5 100644 --- a/test/e2e/storage/testsuites/multivolume.go +++ b/test/e2e/storage/testsuites/multivolume.go @@ -30,7 +30,6 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2epv "k8s.io/kubernetes/test/e2e/framework/pv" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" "k8s.io/kubernetes/test/e2e/storage/testpatterns" @@ -405,8 +404,9 @@ func testAccessMultipleVolumes(f *framework.Framework, cs clientset.Interface, n podConfig := e2epod.Config{ NS: ns, PVCs: pvcs, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), NodeSelection: node, + ImageID: e2evolume.GetDefaultTestImageID(), } pod, err := e2epod.CreateSecPodWithNodeSelection(cs, &podConfig, framework.PodStartTimeout) defer func() { @@ -482,11 +482,11 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int ginkgo.By(fmt.Sprintf("Creating pod%d with a volume on %+v", index, node)) podConfig := e2epod.Config{ NS: ns, - ImageID: imageutils.DebianIptables, PVCs: []*v1.PersistentVolumeClaim{pvc}, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), NodeSelection: node, PVCsReadOnly: readOnly, + ImageID: e2evolume.GetTestImageID(imageutils.DebianIptables), } pod, err := e2epod.CreateSecPodWithNodeSelection(cs, &podConfig, framework.PodStartTimeout) defer func() { @@ -649,8 +649,9 @@ func initializeVolume(cs clientset.Interface, ns string, pvc *v1.PersistentVolum podConfig := e2epod.Config{ NS: ns, PVCs: []*v1.PersistentVolumeClaim{pvc}, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), NodeSelection: node, + ImageID: e2evolume.GetDefaultTestImageID(), } pod, err := e2epod.CreateSecPod(cs, &podConfig, framework.PodStartTimeout) defer func() { diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index c45b952600e..a46deb1e6d7 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -666,7 +666,7 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command Containers: []v1.Container{ { Name: "volume-tester", - Image: e2evolume.GetTestImage(framework.BusyBoxImage), + Image: e2evolume.GetDefaultTestImage(), Command: e2evolume.GenerateScriptCmd(command), VolumeMounts: []v1.VolumeMount{ { diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index 46006a6c4f4..a432bd7f813 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -443,7 +443,7 @@ func (s *subPathTestSuite) DefineTests(driver TestDriver, pattern testpatterns.T defer cleanup() // Change volume container to busybox so we can exec later - l.pod.Spec.Containers[1].Image = e2evolume.GetTestImage(imageutils.GetE2EImage(imageutils.BusyBox)) + l.pod.Spec.Containers[1].Image = e2evolume.GetDefaultTestImage() l.pod.Spec.Containers[1].Command = e2evolume.GenerateScriptCmd("sleep 100000") l.pod.Spec.Containers[1].Args = nil @@ -541,7 +541,7 @@ func SubpathTestPod(f *framework.Framework, subpath, volumeType string, source * InitContainers: []v1.Container{ { Name: fmt.Sprintf("init-volume-%s", suffix), - Image: e2evolume.GetTestImage(imageutils.GetE2EImage(imageutils.BusyBox)), + Image: e2evolume.GetDefaultTestImage(), VolumeMounts: []v1.VolumeMount{volumeMount, probeMount}, SecurityContext: e2evolume.GenerateSecurityContext(privilegedSecurityContext), }, @@ -610,7 +610,7 @@ func volumeFormatPod(f *framework.Framework, volumeSource *v1.VolumeSource) *v1. Containers: []v1.Container{ { Name: fmt.Sprintf("init-volume-%s", f.Namespace.Name), - Image: e2evolume.GetTestImage(imageutils.GetE2EImage(imageutils.BusyBox)), + Image: e2evolume.GetDefaultTestImage(), Command: e2evolume.GenerateScriptCmd("echo nothing"), VolumeMounts: []v1.VolumeMount{ { @@ -789,10 +789,10 @@ func (h *podContainerRestartHooks) FixLivenessProbe(pod *v1.Pod, probeFilePath s func testPodContainerRestartWithHooks(f *framework.Framework, pod *v1.Pod, hooks *podContainerRestartHooks) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure - pod.Spec.Containers[0].Image = e2evolume.GetTestImage(imageutils.GetE2EImage(imageutils.BusyBox)) + pod.Spec.Containers[0].Image = e2evolume.GetDefaultTestImage() pod.Spec.Containers[0].Command = e2evolume.GenerateScriptCmd("sleep 100000") pod.Spec.Containers[0].Args = nil - pod.Spec.Containers[1].Image = e2evolume.GetTestImage(imageutils.GetE2EImage(imageutils.BusyBox)) + pod.Spec.Containers[1].Image = e2evolume.GetDefaultTestImage() pod.Spec.Containers[1].Command = e2evolume.GenerateScriptCmd("sleep 100000") pod.Spec.Containers[1].Args = nil hooks.AddLivenessProbe(pod, probeFilePath) @@ -963,10 +963,10 @@ func testSubpathReconstruction(f *framework.Framework, hostExec utils.HostExec, } // Change to busybox - pod.Spec.Containers[0].Image = e2evolume.GetTestImage(imageutils.GetE2EImage(imageutils.BusyBox)) + pod.Spec.Containers[0].Image = e2evolume.GetDefaultTestImage() pod.Spec.Containers[0].Command = e2evolume.GenerateScriptCmd("sleep 100000") pod.Spec.Containers[0].Args = nil - pod.Spec.Containers[1].Image = e2evolume.GetTestImage(imageutils.GetE2EImage(imageutils.BusyBox)) + pod.Spec.Containers[1].Image = e2evolume.GetDefaultTestImage() pod.Spec.Containers[1].Command = e2evolume.GenerateScriptCmd("sleep 100000") pod.Spec.Containers[1].Args = nil // If grace period is too short, then there is not enough time for the volume diff --git a/test/e2e/storage/testsuites/volume_expand.go b/test/e2e/storage/testsuites/volume_expand.go index 6f213dfb2cf..d6ebac20e96 100644 --- a/test/e2e/storage/testsuites/volume_expand.go +++ b/test/e2e/storage/testsuites/volume_expand.go @@ -32,11 +32,9 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2epv "k8s.io/kubernetes/test/e2e/framework/pv" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" "k8s.io/kubernetes/test/e2e/storage/testpatterns" - imageutils "k8s.io/kubernetes/test/utils/image" ) const ( @@ -177,9 +175,9 @@ func (v *volumeExpandTestSuite) DefineTests(driver TestDriver, pattern testpatte podConfig := e2epod.Config{ NS: f.Namespace.Name, PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc}, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), NodeSelection: l.config.ClientNodeSelection, - ImageID: getTestImage(), + ImageID: e2evolume.GetDefaultTestImageID(), } l.pod, err = e2epod.CreateSecPodWithNodeSelection(f.ClientSet, &podConfig, framework.PodStartTimeout) defer func() { @@ -221,9 +219,9 @@ func (v *volumeExpandTestSuite) DefineTests(driver TestDriver, pattern testpatte podConfig = e2epod.Config{ NS: f.Namespace.Name, PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc}, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), NodeSelection: l.config.ClientNodeSelection, - ImageID: getTestImage(), + ImageID: e2evolume.GetDefaultTestImageID(), } l.pod2, err = e2epod.CreateSecPodWithNodeSelection(f.ClientSet, &podConfig, resizedPodStartupTimeout) defer func() { @@ -249,9 +247,9 @@ func (v *volumeExpandTestSuite) DefineTests(driver TestDriver, pattern testpatte podConfig := e2epod.Config{ NS: f.Namespace.Name, PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc}, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), NodeSelection: l.config.ClientNodeSelection, - ImageID: getTestImage(), + ImageID: e2evolume.GetDefaultTestImageID(), } l.pod, err = e2epod.CreateSecPodWithNodeSelection(f.ClientSet, &podConfig, framework.PodStartTimeout) defer func() { @@ -426,12 +424,3 @@ func WaitForFSResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface) (*v1. } return updatedPVC, nil } - -// TODO: after issue https://github.com/kubernetes/kubernetes/issues/81245 is resolved -// this utility can be moved to e2epod -func getTestImage() int { - if framework.NodeOSDistroIs("windows") { - return imageutils.Agnhost - } - return imageutils.BusyBox -} diff --git a/test/e2e/storage/testsuites/volume_io.go b/test/e2e/storage/testsuites/volume_io.go index ed90f67d559..48ea56a2afb 100644 --- a/test/e2e/storage/testsuites/volume_io.go +++ b/test/e2e/storage/testsuites/volume_io.go @@ -141,7 +141,7 @@ func (t *volumeIOTestSuite) DefineTests(driver TestDriver, pattern testpatterns. l.migrationCheck.validateMigrationVolumeOpCounts() } - ginkgo.It("should write files of various sizes, verify size, validate content [Slow]", func() { + ginkgo.It("should write files of various sizes, verify size, validate content [Slow][LinuxOnly]", func() { init() defer cleanup() @@ -197,7 +197,7 @@ func makePodSpec(config e2evolume.TestConfig, initCmd string, volsrc v1.VolumeSo InitContainers: []v1.Container{ { Name: config.Prefix + "-io-init", - Image: framework.BusyBoxImage, + Image: e2evolume.GetDefaultTestImage(), Command: []string{ "/bin/sh", "-c", @@ -214,7 +214,7 @@ func makePodSpec(config e2evolume.TestConfig, initCmd string, volsrc v1.VolumeSo Containers: []v1.Container{ { Name: config.Prefix + "-io-client", - Image: framework.BusyBoxImage, + Image: e2evolume.GetDefaultTestImage(), Command: []string{ "/bin/sh", "-c", diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index 3e82ad7b613..7e962b983f4 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -215,8 +215,9 @@ func (t *volumeModeTestSuite) DefineTests(driver TestDriver, pattern testpattern podConfig := e2epod.Config{ NS: l.ns.Name, PVCs: []*v1.PersistentVolumeClaim{l.Pvc}, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), NodeSelection: l.config.ClientNodeSelection, + ImageID: e2evolume.GetDefaultTestImageID(), } pod, err := e2epod.MakeSecPod(&podConfig) framework.ExpectNoError(err, "Failed to create pod") @@ -300,7 +301,8 @@ func (t *volumeModeTestSuite) DefineTests(driver TestDriver, pattern testpattern podConfig := e2epod.Config{ NS: l.ns.Name, PVCs: []*v1.PersistentVolumeClaim{l.Pvc}, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), + ImageID: e2evolume.GetDefaultTestImageID(), } pod, err := e2epod.MakeSecPod(&podConfig) framework.ExpectNoError(err) @@ -356,7 +358,8 @@ func (t *volumeModeTestSuite) DefineTests(driver TestDriver, pattern testpattern podConfig := e2epod.Config{ NS: l.ns.Name, PVCs: []*v1.PersistentVolumeClaim{l.Pvc}, - SeLinuxLabel: e2epv.SELinuxLabel, + SeLinuxLabel: e2evolume.GetLinuxLabel(), + ImageID: e2evolume.GetDefaultTestImageID(), } pod, err := e2epod.MakeSecPod(&podConfig) framework.ExpectNoError(err) diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index cb8cccbd0d3..9599895583c 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -227,7 +227,7 @@ func testScriptInPod( Containers: []v1.Container{ { Name: fmt.Sprintf("exec-container-%s", suffix), - Image: e2evolume.GetTestImage(imageutils.GetE2EImage(imageutils.Nginx)), + Image: e2evolume.GetTestImage(imageutils.Nginx), Command: command, VolumeMounts: []v1.VolumeMount{ { diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index e3a4fec83ec..cd0dfdb00ea 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -69,14 +69,18 @@ const ( // PodExec runs f.ExecCommandInContainerWithFullOutput to execute a shell cmd in target pod func PodExec(f *framework.Framework, pod *v1.Pod, shExec string) (string, string, error) { - stdout, stderr, err := f.ExecCommandInContainerWithFullOutput(pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", shExec) - return stdout, stderr, err + if framework.NodeOSDistroIs("windows") { + return f.ExecCommandInContainerWithFullOutput(pod.Name, pod.Spec.Containers[0].Name, "powershell", "/c", shExec) + } + return f.ExecCommandInContainerWithFullOutput(pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", shExec) + } // VerifyExecInPodSucceed verifies shell cmd in target pod succeed func VerifyExecInPodSucceed(f *framework.Framework, pod *v1.Pod, shExec string) { stdout, stderr, err := PodExec(f, pod, shExec) if err != nil { + if exiterr, ok := err.(uexec.CodeExitError); ok { exitCode := exiterr.ExitStatus() framework.ExpectNoError(err, @@ -636,13 +640,19 @@ func CheckReadWriteToPath(f *framework.Framework, pod *v1.Pod, volMode v1.Persis // text -> file1 (write to file) VerifyExecInPodSucceed(f, pod, fmt.Sprintf("echo 'Hello world.' > %s/file1.txt", path)) // grep file1 (read from file and check contents) - VerifyExecInPodSucceed(f, pod, fmt.Sprintf("grep 'Hello world.' %s/file1.txt", path)) - + VerifyExecInPodSucceed(f, pod, readFile("Hello word.", path)) // Check that writing to directory as block volume fails VerifyExecInPodFail(f, pod, fmt.Sprintf("dd if=/dev/urandom of=%s bs=64 count=1", path), 1) } } +func readFile(content, path string) string { + if framework.NodeOSDistroIs("windows") { + return fmt.Sprintf("Select-String '%s' %s/file1.txt", content, path) + } + return fmt.Sprintf("grep 'Hello world.' %s/file1.txt", path) +} + // genBinDataFromSeed generate binData with random seed func genBinDataFromSeed(len int, seed int64) []byte { binData := make([]byte, len)