From 18be0a49e649e71215891f7a78bc2d8527b53cb0 Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Wed, 27 Feb 2019 10:22:35 -0800 Subject: [PATCH] Add GCE PD tests for windows cluster This PR is the first one to add a few GCE PD tests for windows cluster. Will add more tests in later PRs --- test/e2e/framework/volume_util.go | 164 ++++++++++++++----- test/e2e/storage/drivers/csi.go | 6 + test/e2e/storage/drivers/in_tree.go | 45 +++-- test/e2e/storage/testpatterns/testpattern.go | 24 +++ test/e2e/storage/testsuites/base.go | 3 + test/e2e/storage/testsuites/provisioning.go | 12 +- test/e2e/storage/testsuites/volumes.go | 20 ++- 7 files changed, 210 insertions(+), 64 deletions(-) diff --git a/test/e2e/framework/volume_util.go b/test/e2e/framework/volume_util.go index 52f6cf42526..45fe4f1ce6b 100644 --- a/test/e2e/framework/volume_util.go +++ b/test/e2e/framework/volume_util.go @@ -41,6 +41,7 @@ package framework import ( "fmt" + "path/filepath" "strconv" "time" @@ -397,13 +398,21 @@ func VolumeTestCleanup(f *Framework, config VolumeTestConfig) { } } -// Start a client pod using given VolumeSource (exported by startVolumeServer()) +// TestVolumeClient start a client pod using given VolumeSource (exported by startVolumeServer()) // and check that the pod sees expected data, e.g. from the server pod. // Multiple VolumeTests can be specified to mount multiple volumes to a single // pod. func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGroup *int64, fsType string, tests []VolumeTest) { - By(fmt.Sprint("starting ", config.Prefix, " client")) + By(fmt.Sprint("starting ", config.Prefix, "-client")) var gracePeriod int64 = 1 + var command string + + if !NodeOSDistroIs("windows") { + command = "while true ; do cat /opt/0/index.html ; sleep 2 ; ls -altrh /opt/ ; sleep 2 ; done " + } else { + command = "while(1) {cat /opt/0/index.html ; sleep 2 ; ls /opt/; sleep 2}" + } + seLinuxOptions := &v1.SELinuxOptions{Level: "s0:c0,c1"} clientPod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -419,34 +428,24 @@ func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGro Containers: []v1.Container{ { Name: config.Prefix + "-client", - Image: BusyBoxImage, + Image: GetTestImage(BusyBoxImage), WorkingDir: "/opt", // An imperative and easily debuggable container which reads vol contents for // us to scan in the tests or by eye. // We expect that /opt is empty in the minimal containers which we use in this test. - Command: []string{ - "/bin/sh", - "-c", - "while true ; do cat /opt/0/index.html ; sleep 2 ; ls -altrh /opt/ ; sleep 2 ; done ", - }, + Command: GenerateScriptCmd(command), VolumeMounts: []v1.VolumeMount{}, }, }, TerminationGracePeriodSeconds: &gracePeriod, - SecurityContext: &v1.PodSecurityContext{ - SELinuxOptions: &v1.SELinuxOptions{ - Level: "s0:c0,c1", - }, - }, - Volumes: []v1.Volume{}, - NodeName: config.ClientNodeName, - NodeSelector: config.NodeSelector, + SecurityContext: GeneratePodSecurityContext(fsGroup, seLinuxOptions), + Volumes: []v1.Volume{}, + NodeName: config.ClientNodeName, + NodeSelector: config.NodeSelector, }, } podsNamespacer := client.CoreV1().Pods(config.Namespace) - clientPod.Spec.SecurityContext.FSGroup = fsGroup - for i, test := range tests { volumeName := fmt.Sprintf("%s-%s-%d", config.Prefix, "volume", i) clientPod.Spec.Containers[0].VolumeMounts = append(clientPod.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ @@ -461,30 +460,33 @@ func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGro clientPod, err := podsNamespacer.Create(clientPod) if err != nil { Failf("Failed to create %s pod: %v", clientPod.Name, err) + } ExpectNoError(WaitForPodRunningInNamespace(client, clientPod)) By("Checking that text file contents are perfect.") for i, test := range tests { fileName := fmt.Sprintf("/opt/%d/%s", i, test.File) - _, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"cat", fileName}, test.ExpectedContent, time.Minute) + commands := GenerateReadFileCmd(fileName) + _, err = LookForStringInPodExec(config.Namespace, clientPod.Name, commands, test.ExpectedContent, time.Minute) ExpectNoError(err, "failed: finding the contents of the mounted file %s.", fileName) } + if !NodeOSDistroIs("windows") { + if fsGroup != nil { + By("Checking fsGroup is correct.") + _, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"ls", "-ld", "/opt/0"}, strconv.Itoa(int(*fsGroup)), time.Minute) + ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup)) + } - if fsGroup != nil { - By("Checking fsGroup is correct.") - _, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"ls", "-ld", "/opt/0"}, strconv.Itoa(int(*fsGroup)), time.Minute) - ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup)) - } - - if fsType != "" { - By("Checking fsType is correct.") - _, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"grep", " /opt/0 ", "/proc/mounts"}, fsType, time.Minute) - ExpectNoError(err, "failed: getting the right fsType %s", fsType) + if fsType != "" { + By("Checking fsType is correct.") + _, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"grep", " /opt/0 ", "/proc/mounts"}, fsType, time.Minute) + ExpectNoError(err, "failed: getting the right fsType %s", fsType) + } } } -// Insert index.html with given content into given volume. It does so by +// InjectHtml insert index.html with given content into given volume. It does so by // starting and auxiliary pod which writes the file there. // The volume must be writable. func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *int64, volume v1.VolumeSource, content string) { @@ -492,7 +494,7 @@ func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *in podClient := client.CoreV1().Pods(config.Namespace) podName := fmt.Sprintf("%s-injector-%s", config.Prefix, rand.String(4)) volMountName := fmt.Sprintf("%s-volume-%s", config.Prefix, rand.String(4)) - privileged := true + fileName := "/mnt/index.html" injectPod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ @@ -509,18 +511,15 @@ func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *in Containers: []v1.Container{ { Name: config.Prefix + "-injector", - Image: BusyBoxImage, - Command: []string{"/bin/sh"}, - Args: []string{"-c", "echo '" + content + "' > /mnt/index.html && chmod o+rX /mnt /mnt/index.html"}, + Image: GetTestImage(BusyBoxImage), + Command: GenerateWriteFileCmd(content, fileName), VolumeMounts: []v1.VolumeMount{ { Name: volMountName, MountPath: "/mnt", }, }, - SecurityContext: &v1.SecurityContext{ - Privileged: &privileged, - }, + SecurityContext: GenerateSecurityContext(true), }, }, SecurityContext: &v1.PodSecurityContext{ @@ -559,3 +558,94 @@ func CreateGCEVolume() (*v1.PersistentVolumeSource, string) { }, }, diskName } + +// GenerateScriptCmd generates the corresponding command lines to execute a command. +// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh +func GenerateScriptCmd(command string) []string { + var commands []string + if !NodeOSDistroIs("windows") { + commands = []string{"/bin/sh", "-c", command} + } else { + commands = []string{"powershell", "/c", command} + } + return commands +} + +// GenerateWriteFileCmd generates the corresponding command lines to write a file with the given content and file path. +// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh +func GenerateWriteFileCmd(content, fullPath string) []string { + var commands []string + if !NodeOSDistroIs("windows") { + commands = []string{"/bin/sh", "-c", "echo '" + content + "' > " + fullPath} + } else { + commands = []string{"powershell", "/c", "echo '" + content + "' > " + fullPath} + } + return commands +} + +// GenerateReadFileCmd generates the corresponding command lines to read from a file with the given file path. +// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh +func GenerateReadFileCmd(fullPath string) []string { + var commands []string + if !NodeOSDistroIs("windows") { + commands = []string{"cat", fullPath} + } else { + commands = []string{"powershell", "/c", "type " + fullPath} + } + return commands +} + +// GenerateWriteandExecuteScriptFileCmd generates the corresponding command lines to write a file with the given file path +// and also execute this file. +// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh +func GenerateWriteandExecuteScriptFileCmd(content, fileName, filePath string) []string { + // for windows cluster, modify the Pod spec. + if NodeOSDistroIs("windows") { + scriptName := fmt.Sprintf("%s.ps1", fileName) + fullPath := filepath.Join(filePath, scriptName) + + cmd := "echo \"" + content + "\" > " + fullPath + "; .\\" + fullPath + Logf("generated pod command %s", cmd) + return []string{"powershell", "/c", cmd} + } else { + scriptName := fmt.Sprintf("%s.sh", fileName) + fullPath := filepath.Join(filePath, scriptName) + cmd := fmt.Sprintf("echo \"%s\" > %s; chmod u+x %s; %s;", content, fullPath, fullPath, fullPath) + return []string{"/bin/sh", "-ec", cmd} + } +} + +// GenerateSecurityContext generates the corresponding container security context with the given inputs +// If the Node OS is windows, currently we will ignore the inputs and return nil. +// TODO: Will modify it after windows has its own security context +func GenerateSecurityContext(privileged bool) *v1.SecurityContext { + if NodeOSDistroIs("windows") { + return nil + } + return &v1.SecurityContext{ + Privileged: &privileged, + } +} + +// GeneratePodSecurityContext generates the corresponding pod security context with the given inputs +// If the Node OS is windows, currently we will ignore the inputs and return nil. +// TODO: Will modify it after windows has its own security context +func GeneratePodSecurityContext(fsGroup *int64, seLinuxOptions *v1.SELinuxOptions) *v1.PodSecurityContext { + if NodeOSDistroIs("windows") { + return nil + } + return &v1.PodSecurityContext{ + SELinuxOptions: seLinuxOptions, + FSGroup: fsGroup, + } +} + +// GetTestImage returns the image name with the given input +// If the Node OS is windows, currently we return Nettest image for Windows node +// due to the issue of #https://github.com/kubernetes-sigs/windows-testing/pull/35. +func GetTestImage(image string) string { + if NodeOSDistroIs("windows") { + return imageutils.GetE2EImage(imageutils.Nettest) + } + return image +} diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 7eb310cf027..d802bfe45ae 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -347,6 +347,9 @@ func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { if pattern.FsType == "xfs" { framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom") } + if pattern.FeatureTag == "sig-windows" { + framework.Skipf("Skipping tests for windows since CSI does not support it yet") + } } func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass { @@ -461,6 +464,9 @@ func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPa if pattern.FsType == "xfs" { framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom") } + if pattern.FeatureTag == "sig-windows" { + framework.Skipf("Skipping tests for windows since CSI does not support it yet") + } } func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass { diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 2cde7f2ea6b..886a00a470a 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -1104,23 +1104,32 @@ var _ testsuites.InlineVolumeTestDriver = &gcePdDriver{} var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{} var _ testsuites.DynamicPVTestDriver = &gcePdDriver{} -// InitGceDriver returns gcePdDriver that implements TestDriver interface +// InitGcePdDriver returns gcePdDriver that implements TestDriver interface func InitGcePdDriver() testsuites.TestDriver { + var supportedTypes sets.String + var capFsGroup bool + if framework.NodeOSDistroIs("windows") { + supportedTypes = sets.NewString("ntfs") + capFsGroup = false + } else { + supportedTypes = sets.NewString( + "", // Default fsType + "ext2", + "ext3", + "ext4", + "xfs", + ) + capFsGroup = true + } return &gcePdDriver{ driverInfo: testsuites.DriverInfo{ - Name: "gcepd", - MaxFileSize: testpatterns.FileSizeMedium, - SupportedFsType: sets.NewString( - "", // Default fsType - "ext2", - "ext3", - "ext4", - "xfs", - ), + Name: "gcepd", + MaxFileSize: testpatterns.FileSizeMedium, + SupportedFsType: supportedTypes, SupportedMountOption: sets.NewString("debug", "nouid32"), Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, - testsuites.CapFsGroup: true, + testsuites.CapFsGroup: capFsGroup, testsuites.CapBlock: true, testsuites.CapExec: true, }, @@ -1134,6 +1143,9 @@ func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo { func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { framework.SkipUnlessProviderIs("gce", "gke") + if pattern.FeatureTag == "sig-windows" { + framework.SkipUnlessNodeOSDistroIs("windows") + } } func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, volume testsuites.TestVolume) *v1.VolumeSource { @@ -1183,11 +1195,18 @@ func (h *gcePdDriver) GetClaimSize() string { } func (g *gcePdDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) { - return &testsuites.PerTestConfig{ + config := &testsuites.PerTestConfig{ Driver: g, Prefix: "gcepd", Framework: f, - }, func() {} + } + if framework.NodeOSDistroIs("windows") { + config.ClientNodeSelector = map[string]string{ + "beta.kubernetes.io/os": "windows", + } + } + return config, func() {} + } func (g *gcePdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume { diff --git a/test/e2e/storage/testpatterns/testpattern.go b/test/e2e/storage/testpatterns/testpattern.go index a9fc2df37c2..410ab85f4fe 100644 --- a/test/e2e/storage/testpatterns/testpattern.go +++ b/test/e2e/storage/testpatterns/testpattern.go @@ -145,6 +145,30 @@ var ( FsType: "xfs", } + // Definitions for ntfs + + // NtfsInlineVolume is TestPattern for "Inline-volume (ntfs)" + NtfsInlineVolume = TestPattern{ + Name: "Inline-volume (ntfs)", + VolType: InlineVolume, + FsType: "ntfs", + FeatureTag: "sig-windows", + } + // NtfsPreprovisionedPV is TestPattern for "Pre-provisioned PV (ntfs)" + NtfsPreprovisionedPV = TestPattern{ + Name: "Pre-provisioned PV (ntfs)", + VolType: PreprovisionedPV, + FsType: "ntfs", + FeatureTag: "sig-windows", + } + // NtfsDynamicPV is TestPattern for "Dynamic PV (xfs)" + NtfsDynamicPV = TestPattern{ + Name: "Dynamic PV (ntfs)", + VolType: DynamicPV, + FsType: "ntfs", + FeatureTag: "sig-windows", + } + // Definitions for Filesystem volume mode // FsVolModePreprovisionedPV is TestPattern for "Pre-provisioned PV (filesystem)" diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 9cd5dd499bb..a7326a9541f 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -133,6 +133,9 @@ func skipUnsupportedTest(driver TestDriver, pattern testpatterns.TestPattern) { if pattern.FsType == "xfs" && framework.NodeOSDistroIs("gci") { framework.Skipf("Distro doesn't support xfs -- skipping") } + if pattern.FsType == "ntfs" && !framework.NodeOSDistroIs("windows") { + framework.Skipf("Distro %s doesn't support ntfs -- skipping", framework.TestContext.NodeOSDistro) + } } // 4. Check with driver specific logic diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 9bca03aa78d..cefce04b3cc 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -35,7 +35,6 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/testpatterns" - imageutils "k8s.io/kubernetes/test/utils/image" ) // StorageClassTest represents parameters to be used by provisioning tests. @@ -70,6 +69,7 @@ func InitProvisioningTestSuite() TestSuite { name: "provisioning", testPatterns: []testpatterns.TestPattern{ testpatterns.DefaultFsDynamicPV, + testpatterns.NtfsDynamicPV, }, }, } @@ -120,7 +120,7 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte l.config, l.testCleanup = driver.PrepareTest(f) l.cs = l.config.Framework.ClientSet claimSize := dDriver.GetClaimSize() - l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, "") + l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, pattern.FsType) if l.sc == nil { framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name) } @@ -472,6 +472,9 @@ func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClai By(fmt.Sprintf("checking the created volume is readable and retains data on another node %+v", secondNode)) command = "grep 'hello world' /mnt/test/data" + if framework.NodeOSDistroIs("windows") { + command = "select-string 'hello world' /mnt/test/data" + } pod = StartInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-reader-node2", command, secondNode) framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(client, pod.Name, pod.Namespace)) runningPod, err = client.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) @@ -607,9 +610,8 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command Containers: []v1.Container{ { Name: "volume-tester", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"/bin/sh"}, - Args: []string{"-c", command}, + Image: framework.GetTestImage(framework.BusyBoxImage), + Command: framework.GenerateScriptCmd(command), VolumeMounts: []v1.VolumeMount{ { Name: "my-volume", diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index 364a1bd5016..957b4e5cd13 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -23,7 +23,6 @@ package testsuites import ( "fmt" - "path/filepath" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -63,6 +62,10 @@ func InitVolumesTestSuite() TestSuite { testpatterns.XfsInlineVolume, testpatterns.XfsPreprovisionedPV, testpatterns.XfsDynamicPV, + // ntfs + testpatterns.NtfsInlineVolume, + testpatterns.NtfsPreprovisionedPV, + testpatterns.NtfsDynamicPV, }, }, } @@ -105,7 +108,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T // registers its own BeforeEach which creates the namespace. Beware that it // also registers an AfterEach which renders f unusable. Any code using // f must run inside an It or Context callback. - f := framework.NewDefaultFramework("volumeio") + f := framework.NewDefaultFramework("volume") init := func() { l = local{} @@ -181,10 +184,9 @@ func testScriptInPod( volName = "vol1" ) suffix := generateSuffixForPodName(volumeType) - scriptName := fmt.Sprintf("test-%s.sh", suffix) - fullPath := filepath.Join(volPath, scriptName) - cmd := fmt.Sprintf("echo \"ls %s\" > %s; chmod u+x %s; %s", volPath, fullPath, fullPath, fullPath) - + fileName := fmt.Sprintf("test-%s", suffix) + content := fmt.Sprintf("ls %s", volPath) + command := framework.GenerateWriteandExecuteScriptFileCmd(content, fileName, volPath) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("exec-volume-test-%s", suffix), @@ -194,8 +196,8 @@ func testScriptInPod( Containers: []v1.Container{ { Name: fmt.Sprintf("exec-container-%s", suffix), - Image: imageutils.GetE2EImage(imageutils.Nginx), - Command: []string{"/bin/sh", "-ec", cmd}, + Image: framework.GetTestImage(imageutils.GetE2EImage(imageutils.Nginx)), + Command: command, VolumeMounts: []v1.VolumeMount{ { Name: volName, @@ -215,7 +217,7 @@ func testScriptInPod( }, } By(fmt.Sprintf("Creating pod %s", pod.Name)) - f.TestContainerOutput("exec-volume-test", pod, 0, []string{scriptName}) + f.TestContainerOutput("exec-volume-test", pod, 0, []string{fileName}) By(fmt.Sprintf("Deleting pod %s", pod.Name)) err := framework.DeletePodWithWait(f, f.ClientSet, pod)