Merge pull request #74990 from jingxu97/pd_windows_test

Add GCE PD tests for windows cluster

test/e2e/framework/volume_util.go

@@ -41,6 +41,7 @@ package framework
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"strconv"
 	"time"
@@ -397,13 +398,21 @@ func VolumeTestCleanup(f *Framework, config VolumeTestConfig) {
 	}
 }
 
-// Start a client pod using given VolumeSource (exported by startVolumeServer())
+// TestVolumeClient start a client pod using given VolumeSource (exported by startVolumeServer())
 // and check that the pod sees expected data, e.g. from the server pod.
 // Multiple VolumeTests can be specified to mount multiple volumes to a single
 // pod.
 func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGroup *int64, fsType string, tests []VolumeTest) {
-	By(fmt.Sprint("starting ", config.Prefix, " client"))
+	By(fmt.Sprint("starting ", config.Prefix, "-client"))
 	var gracePeriod int64 = 1
+	var command string
+
+	if !NodeOSDistroIs("windows") {
+		command = "while true ; do cat /opt/0/index.html ; sleep 2 ; ls -altrh /opt/  ; sleep 2 ; done "
+	} else {
+		command = "while(1) {cat /opt/0/index.html ; sleep 2 ; ls /opt/; sleep 2}"
+	}
+	seLinuxOptions := &v1.SELinuxOptions{Level: "s0:c0,c1"}
 	clientPod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Pod",
@@ -419,34 +428,24 @@ func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGro
 			Containers: []v1.Container{
 				{
 					Name:       config.Prefix + "-client",
-					Image:      BusyBoxImage,
+					Image:      GetTestImage(BusyBoxImage),
 					WorkingDir: "/opt",
 					// An imperative and easily debuggable container which reads vol contents for
 					// us to scan in the tests or by eye.
 					// We expect that /opt is empty in the minimal containers which we use in this test.
-					Command: []string{
+					Command:      GenerateScriptCmd(command),
-						"/bin/sh",
-						"-c",
-						"while true ; do cat /opt/0/index.html ; sleep 2 ; ls -altrh /opt/  ; sleep 2 ; done ",
-					},
 					VolumeMounts: []v1.VolumeMount{},
 				},
 			},
 			TerminationGracePeriodSeconds: &gracePeriod,
-			SecurityContext: &v1.PodSecurityContext{
+			SecurityContext:               GeneratePodSecurityContext(fsGroup, seLinuxOptions),
-				SELinuxOptions: &v1.SELinuxOptions{
+			Volumes:                       []v1.Volume{},
-					Level: "s0:c0,c1",
+			NodeName:                      config.ClientNodeName,
-				},
+			NodeSelector:                  config.NodeSelector,
-			},
-			Volumes:      []v1.Volume{},
-			NodeName:     config.ClientNodeName,
-			NodeSelector: config.NodeSelector,
 		},
 	}
 	podsNamespacer := client.CoreV1().Pods(config.Namespace)
 
-	clientPod.Spec.SecurityContext.FSGroup = fsGroup
-
 	for i, test := range tests {
 		volumeName := fmt.Sprintf("%s-%s-%d", config.Prefix, "volume", i)
 		clientPod.Spec.Containers[0].VolumeMounts = append(clientPod.Spec.Containers[0].VolumeMounts, v1.VolumeMount{
@@ -461,30 +460,33 @@ func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGro
 	clientPod, err := podsNamespacer.Create(clientPod)
 	if err != nil {
 		Failf("Failed to create %s pod: %v", clientPod.Name, err)
+
 	}
 	ExpectNoError(WaitForPodRunningInNamespace(client, clientPod))
 
 	By("Checking that text file contents are perfect.")
 	for i, test := range tests {
 		fileName := fmt.Sprintf("/opt/%d/%s", i, test.File)
-		_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"cat", fileName}, test.ExpectedContent, time.Minute)
+		commands := GenerateReadFileCmd(fileName)
+		_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, commands, test.ExpectedContent, time.Minute)
 		ExpectNoError(err, "failed: finding the contents of the mounted file %s.", fileName)
 	}
+	if !NodeOSDistroIs("windows") {
+		if fsGroup != nil {
+			By("Checking fsGroup is correct.")
+			_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"ls", "-ld", "/opt/0"}, strconv.Itoa(int(*fsGroup)), time.Minute)
+			ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup))
+		}
 
-	if fsGroup != nil {
+		if fsType != "" {
-		By("Checking fsGroup is correct.")
+			By("Checking fsType is correct.")
-		_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"ls", "-ld", "/opt/0"}, strconv.Itoa(int(*fsGroup)), time.Minute)
+			_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"grep", " /opt/0 ", "/proc/mounts"}, fsType, time.Minute)
-		ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup))
+			ExpectNoError(err, "failed: getting the right fsType %s", fsType)
-	}
+		}
-
-	if fsType != "" {
-		By("Checking fsType is correct.")
-		_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"grep", " /opt/0 ", "/proc/mounts"}, fsType, time.Minute)
-		ExpectNoError(err, "failed: getting the right fsType %s", fsType)
 	}
 }
 
-// Insert index.html with given content into given volume. It does so by
+// InjectHtml insert index.html with given content into given volume. It does so by
 // starting and auxiliary pod which writes the file there.
 // The volume must be writable.
 func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *int64, volume v1.VolumeSource, content string) {
@@ -492,7 +494,7 @@ func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *in
 	podClient := client.CoreV1().Pods(config.Namespace)
 	podName := fmt.Sprintf("%s-injector-%s", config.Prefix, rand.String(4))
 	volMountName := fmt.Sprintf("%s-volume-%s", config.Prefix, rand.String(4))
-	privileged := true
+	fileName := "/mnt/index.html"
 
 	injectPod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
@@ -509,18 +511,15 @@ func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *in
 			Containers: []v1.Container{
 				{
 					Name:    config.Prefix + "-injector",
-					Image:   BusyBoxImage,
+					Image:   GetTestImage(BusyBoxImage),
-					Command: []string{"/bin/sh"},
+					Command: GenerateWriteFileCmd(content, fileName),
-					Args:    []string{"-c", "echo '" + content + "' > /mnt/index.html && chmod o+rX /mnt /mnt/index.html"},
 					VolumeMounts: []v1.VolumeMount{
 						{
 							Name:      volMountName,
 							MountPath: "/mnt",
 						},
 					},
-					SecurityContext: &v1.SecurityContext{
+					SecurityContext: GenerateSecurityContext(true),
-						Privileged: &privileged,
-					},
 				},
 			},
 			SecurityContext: &v1.PodSecurityContext{
@@ -559,3 +558,94 @@ func CreateGCEVolume() (*v1.PersistentVolumeSource, string) {
 		},
 	}, diskName
 }
+
+// GenerateScriptCmd generates the corresponding command lines to execute a command.
+// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
+func GenerateScriptCmd(command string) []string {
+	var commands []string
+	if !NodeOSDistroIs("windows") {
+		commands = []string{"/bin/sh", "-c", command}
+	} else {
+		commands = []string{"powershell", "/c", command}
+	}
+	return commands
+}
+
+// GenerateWriteFileCmd generates the corresponding command lines to write a file with the given content and file path.
+// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
+func GenerateWriteFileCmd(content, fullPath string) []string {
+	var commands []string
+	if !NodeOSDistroIs("windows") {
+		commands = []string{"/bin/sh", "-c", "echo '" + content + "' > " + fullPath}
+	} else {
+		commands = []string{"powershell", "/c", "echo '" + content + "' > " + fullPath}
+	}
+	return commands
+}
+
+// GenerateReadFileCmd generates the corresponding command lines to read from a file with the given file path.
+// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
+func GenerateReadFileCmd(fullPath string) []string {
+	var commands []string
+	if !NodeOSDistroIs("windows") {
+		commands = []string{"cat", fullPath}
+	} else {
+		commands = []string{"powershell", "/c", "type " + fullPath}
+	}
+	return commands
+}
+
+// GenerateWriteandExecuteScriptFileCmd generates the corresponding command lines to write a file with the given file path
+// and also execute this file.
+// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
+func GenerateWriteandExecuteScriptFileCmd(content, fileName, filePath string) []string {
+	// for windows cluster, modify the Pod spec.
+	if NodeOSDistroIs("windows") {
+		scriptName := fmt.Sprintf("%s.ps1", fileName)
+		fullPath := filepath.Join(filePath, scriptName)
+
+		cmd := "echo \"" + content + "\" > " + fullPath + "; .\\" + fullPath
+		Logf("generated pod command %s", cmd)
+		return []string{"powershell", "/c", cmd}
+	} else {
+		scriptName := fmt.Sprintf("%s.sh", fileName)
+		fullPath := filepath.Join(filePath, scriptName)
+		cmd := fmt.Sprintf("echo \"%s\" > %s; chmod u+x %s; %s;", content, fullPath, fullPath, fullPath)
+		return []string{"/bin/sh", "-ec", cmd}
+	}
+}
+
+// GenerateSecurityContext generates the corresponding container security context with the given inputs
+// If the Node OS is windows, currently we will ignore the inputs and return nil.
+// TODO: Will modify it after windows has its own security context
+func GenerateSecurityContext(privileged bool) *v1.SecurityContext {
+	if NodeOSDistroIs("windows") {
+		return nil
+	}
+	return &v1.SecurityContext{
+		Privileged: &privileged,
+	}
+}
+
+// GeneratePodSecurityContext generates the corresponding pod security context with the given inputs
+// If the Node OS is windows, currently we will ignore the inputs and return nil.
+// TODO: Will modify it after windows has its own security context
+func GeneratePodSecurityContext(fsGroup *int64, seLinuxOptions *v1.SELinuxOptions) *v1.PodSecurityContext {
+	if NodeOSDistroIs("windows") {
+		return nil
+	}
+	return &v1.PodSecurityContext{
+		SELinuxOptions: seLinuxOptions,
+		FSGroup:        fsGroup,
+	}
+}
+
+// GetTestImage returns the image name with the given input
+// If the Node OS is windows, currently we return Nettest image for Windows node
+// due to the issue of #https://github.com/kubernetes-sigs/windows-testing/pull/35.
+func GetTestImage(image string) string {
+	if NodeOSDistroIs("windows") {
+		return imageutils.GetE2EImage(imageutils.Nettest)
+	}
+	return image
+}

test/e2e/storage/drivers/csi.go

@@ -346,6 +346,9 @@ func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
 	if pattern.FsType == "xfs" {
 		framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom")
 	}
+	if pattern.FeatureTag == "sig-windows" {
+		framework.Skipf("Skipping tests for windows since CSI does not support it yet")
+	}
 }
 
 func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
@@ -449,6 +452,9 @@ func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPa
 	if pattern.FsType == "xfs" {
 		framework.SkipUnlessNodeOSDistroIs("ubuntu", "custom")
 	}
+	if pattern.FeatureTag == "sig-windows" {
+		framework.Skipf("Skipping tests for windows since CSI does not support it yet")
+	}
 }
 
 func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {

test/e2e/storage/drivers/in_tree.go

@@ -1104,23 +1104,32 @@ var _ testsuites.InlineVolumeTestDriver = &gcePdDriver{}
 var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{}
 var _ testsuites.DynamicPVTestDriver = &gcePdDriver{}
 
-// InitGceDriver returns gcePdDriver that implements TestDriver interface
+// InitGcePdDriver returns gcePdDriver that implements TestDriver interface
 func InitGcePdDriver() testsuites.TestDriver {
+	var supportedTypes sets.String
+	var capFsGroup bool
+	if framework.NodeOSDistroIs("windows") {
+		supportedTypes = sets.NewString("ntfs")
+		capFsGroup = false
+	} else {
+		supportedTypes = sets.NewString(
+			"", // Default fsType
+			"ext2",
+			"ext3",
+			"ext4",
+			"xfs",
+		)
+		capFsGroup = true
+	}
 	return &gcePdDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "gcepd",
+			Name:                 "gcepd",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:          testpatterns.FileSizeMedium,
-			SupportedFsType: sets.NewString(
+			SupportedFsType:      supportedTypes,
-				"", // Default fsType
-				"ext2",
-				"ext3",
-				"ext4",
-				"xfs",
-			),
 			SupportedMountOption: sets.NewString("debug", "nouid32"),
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
-				testsuites.CapFsGroup:     true,
+				testsuites.CapFsGroup:     capFsGroup,
 				testsuites.CapBlock:       true,
 				testsuites.CapExec:        true,
 			},
@@ -1134,6 +1143,9 @@ func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo {
 
 func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
 	framework.SkipUnlessProviderIs("gce", "gke")
+	if pattern.FeatureTag == "sig-windows" {
+		framework.SkipUnlessNodeOSDistroIs("windows")
+	}
 }
 
 func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, volume testsuites.TestVolume) *v1.VolumeSource {
@@ -1183,11 +1195,18 @@ func (h *gcePdDriver) GetClaimSize() string {
 }
 
 func (g *gcePdDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
-	return &testsuites.PerTestConfig{
+	config := &testsuites.PerTestConfig{
 		Driver:    g,
 		Prefix:    "gcepd",
 		Framework: f,
-	}, func() {}
+	}
+	if framework.NodeOSDistroIs("windows") {
+		config.ClientNodeSelector = map[string]string{
+			"beta.kubernetes.io/os": "windows",
+		}
+	}
+	return config, func() {}
+
 }
 
 func (g *gcePdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {

test/e2e/storage/testpatterns/testpattern.go

@@ -145,6 +145,30 @@ var (
 		FsType:  "xfs",
 	}
 
+	// Definitions for ntfs
+
+	// NtfsInlineVolume is TestPattern for "Inline-volume (ntfs)"
+	NtfsInlineVolume = TestPattern{
+		Name:       "Inline-volume (ntfs)",
+		VolType:    InlineVolume,
+		FsType:     "ntfs",
+		FeatureTag: "sig-windows",
+	}
+	// NtfsPreprovisionedPV is TestPattern for "Pre-provisioned PV (ntfs)"
+	NtfsPreprovisionedPV = TestPattern{
+		Name:       "Pre-provisioned PV (ntfs)",
+		VolType:    PreprovisionedPV,
+		FsType:     "ntfs",
+		FeatureTag: "sig-windows",
+	}
+	// NtfsDynamicPV is TestPattern for "Dynamic PV (xfs)"
+	NtfsDynamicPV = TestPattern{
+		Name:       "Dynamic PV (ntfs)",
+		VolType:    DynamicPV,
+		FsType:     "ntfs",
+		FeatureTag: "sig-windows",
+	}
+
 	// Definitions for Filesystem volume mode
 
 	// FsVolModePreprovisionedPV is TestPattern for "Pre-provisioned PV (filesystem)"

test/e2e/storage/testsuites/base.go

@@ -133,6 +133,9 @@ func skipUnsupportedTest(driver TestDriver, pattern testpatterns.TestPattern) {
 		if pattern.FsType == "xfs" && framework.NodeOSDistroIs("gci") {
 			framework.Skipf("Distro doesn't support xfs -- skipping")
 		}
+		if pattern.FsType == "ntfs" && !framework.NodeOSDistroIs("windows") {
+			framework.Skipf("Distro %s doesn't support ntfs -- skipping", framework.TestContext.NodeOSDistro)
+		}
 	}
 
 	// 4. Check with driver specific logic

test/e2e/storage/testsuites/provisioning.go

@@ -35,7 +35,6 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
-	imageutils "k8s.io/kubernetes/test/utils/image"
 )
 
 // StorageClassTest represents parameters to be used by provisioning tests.
@@ -70,6 +69,7 @@ func InitProvisioningTestSuite() TestSuite {
 			name: "provisioning",
 			testPatterns: []testpatterns.TestPattern{
 				testpatterns.DefaultFsDynamicPV,
+				testpatterns.NtfsDynamicPV,
 			},
 		},
 	}
@@ -120,7 +120,7 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
 		l.config, l.testCleanup = driver.PrepareTest(f)
 		l.cs = l.config.Framework.ClientSet
 		claimSize := dDriver.GetClaimSize()
-		l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, "")
+		l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, pattern.FsType)
 		if l.sc == nil {
 			framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name)
 		}
@@ -472,6 +472,9 @@ func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClai
 
 	By(fmt.Sprintf("checking the created volume is readable and retains data on another node %+v", secondNode))
 	command = "grep 'hello world' /mnt/test/data"
+	if framework.NodeOSDistroIs("windows") {
+		command = "select-string 'hello world' /mnt/test/data"
+	}
 	pod = StartInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-reader-node2", command, secondNode)
 	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(client, pod.Name, pod.Namespace))
 	runningPod, err = client.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
@@ -607,9 +610,8 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command
 			Containers: []v1.Container{
 				{
 					Name:    "volume-tester",
-					Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+					Image:   framework.GetTestImage(framework.BusyBoxImage),
-					Command: []string{"/bin/sh"},
+					Command: framework.GenerateScriptCmd(command),
-					Args:    []string{"-c", command},
 					VolumeMounts: []v1.VolumeMount{
 						{
 							Name:      "my-volume",

test/e2e/storage/testsuites/volumes.go

@@ -23,7 +23,6 @@ package testsuites
 
 import (
 	"fmt"
-	"path/filepath"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -63,6 +62,10 @@ func InitVolumesTestSuite() TestSuite {
 				testpatterns.XfsInlineVolume,
 				testpatterns.XfsPreprovisionedPV,
 				testpatterns.XfsDynamicPV,
+				// ntfs
+				testpatterns.NtfsInlineVolume,
+				testpatterns.NtfsPreprovisionedPV,
+				testpatterns.NtfsDynamicPV,
 			},
 		},
 	}
@@ -105,7 +108,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 	// registers its own BeforeEach which creates the namespace. Beware that it
 	// also registers an AfterEach which renders f unusable. Any code using
 	// f must run inside an It or Context callback.
-	f := framework.NewDefaultFramework("volumeio")
+	f := framework.NewDefaultFramework("volume")
 
 	init := func() {
 		l = local{}
@@ -181,10 +184,9 @@ func testScriptInPod(
 		volName = "vol1"
 	)
 	suffix := generateSuffixForPodName(volumeType)
-	scriptName := fmt.Sprintf("test-%s.sh", suffix)
+	fileName := fmt.Sprintf("test-%s", suffix)
-	fullPath := filepath.Join(volPath, scriptName)
+	content := fmt.Sprintf("ls %s", volPath)
-	cmd := fmt.Sprintf("echo \"ls %s\" > %s; chmod u+x %s; %s", volPath, fullPath, fullPath, fullPath)
+	command := framework.GenerateWriteandExecuteScriptFileCmd(content, fileName, volPath)
-
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("exec-volume-test-%s", suffix),
@@ -194,8 +196,8 @@ func testScriptInPod(
 			Containers: []v1.Container{
 				{
 					Name:    fmt.Sprintf("exec-container-%s", suffix),
-					Image:   imageutils.GetE2EImage(imageutils.Nginx),
+					Image:   framework.GetTestImage(imageutils.GetE2EImage(imageutils.Nginx)),
-					Command: []string{"/bin/sh", "-ec", cmd},
+					Command: command,
 					VolumeMounts: []v1.VolumeMount{
 						{
 							Name:      volName,
@@ -215,7 +217,7 @@ func testScriptInPod(
 		},
 	}
 	By(fmt.Sprintf("Creating pod %s", pod.Name))
-	f.TestContainerOutput("exec-volume-test", pod, 0, []string{scriptName})
+	f.TestContainerOutput("exec-volume-test", pod, 0, []string{fileName})
 
 	By(fmt.Sprintf("Deleting pod %s", pod.Name))
 	err := framework.DeletePodWithWait(f, f.ClientSet, pod)