From fb9f02b5e11ad89fa6f1ca879683b7aed84c07ee Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 11 Feb 2020 19:17:41 -0800 Subject: [PATCH] Don't set NodeName directly in Pods so that it still goes through the scheduler Change-Id: I244b6aac0289a13339f3ac228c4ad9ecf8c07b42 --- test/e2e/framework/pod/create.go | 5 +---- test/e2e/framework/pod/node_selection.go | 16 ++++++++++++++++ test/e2e/storage/persistent_volumes-local.go | 3 ++- test/e2e/storage/testsuites/ephemeral.go | 4 +--- test/e2e/storage/testsuites/provisioning.go | 4 +--- test/e2e/storage/testsuites/subpath.go | 11 +++++++---- test/e2e/storage/testsuites/volume_io.go | 10 +++++++--- test/e2e/storage/testsuites/volumemode.go | 2 +- test/e2e/storage/testsuites/volumes.go | 6 ++++-- 9 files changed, 40 insertions(+), 21 deletions(-) diff --git a/test/e2e/framework/pod/create.go b/test/e2e/framework/pod/create.go index af7122ed3b0..1819d0b587a 100644 --- a/test/e2e/framework/pod/create.go +++ b/test/e2e/framework/pod/create.go @@ -86,10 +86,7 @@ func CreateSecPod(client clientset.Interface, namespace string, pvclaims []*v1.P // CreateSecPodWithNodeSelection creates security pod with given claims func CreateSecPodWithNodeSelection(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, inlineVolumeSources []*v1.VolumeSource, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, node NodeSelection, timeout time.Duration) (*v1.Pod, error) { pod := MakeSecPod(namespace, pvclaims, inlineVolumeSources, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup) - // Setting node - pod.Spec.NodeName = node.Name - pod.Spec.NodeSelector = node.Selector - pod.Spec.Affinity = node.Affinity + SetNodeSelection(pod, node) pod, err := client.CoreV1().Pods(namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) if err != nil { diff --git a/test/e2e/framework/pod/node_selection.go b/test/e2e/framework/pod/node_selection.go index 01d1d4c165c..50d530ee705 100644 --- a/test/e2e/framework/pod/node_selection.go +++ b/test/e2e/framework/pod/node_selection.go @@ -87,3 +87,19 @@ func SetNodeAffinity(pod *v1.Pod, nodeName string) { SetAffinity(nodeSelection, nodeName) pod.Spec.Affinity = nodeSelection.Affinity } + +// SetNodeSelection modifies the given pod object with +// the specified NodeSelection +func SetNodeSelection(pod *v1.Pod, nodeSelection NodeSelection) { + pod.Spec.NodeSelector = nodeSelection.Selector + pod.Spec.Affinity = nodeSelection.Affinity + // pod.Spec.NodeName should not be set directly because + // it will bypass the scheduler, potentially causing + // kubelet to Fail the pod immediately if it's out of + // resources. Instead, we want the pod to remain + // pending in the scheduler until the node has resources + // freed up. + if nodeSelection.Name != "" { + SetNodeAffinity(pod, nodeSelection.Name) + } +} diff --git a/test/e2e/storage/persistent_volumes-local.go b/test/e2e/storage/persistent_volumes-local.go index f2bc0fd8a93..69ba10db7c2 100644 --- a/test/e2e/storage/persistent_volumes-local.go +++ b/test/e2e/storage/persistent_volumes-local.go @@ -988,7 +988,8 @@ func makeLocalPodWithNodeName(config *localTestConfig, volume *localTestVolume, if pod == nil { return } - pod.Spec.NodeName = nodeName + + e2epod.SetNodeAffinity(pod, nodeName) return } diff --git a/test/e2e/storage/testsuites/ephemeral.go b/test/e2e/storage/testsuites/ephemeral.go index c52f02a3b1d..c94aaaccf24 100644 --- a/test/e2e/storage/testsuites/ephemeral.go +++ b/test/e2e/storage/testsuites/ephemeral.go @@ -291,9 +291,6 @@ func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command stri }, }, Spec: v1.PodSpec{ - NodeName: node.Name, - NodeSelector: node.Selector, - Affinity: node.Affinity, Containers: []v1.Container{ { Name: "csi-volume-tester", @@ -304,6 +301,7 @@ func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command stri RestartPolicy: v1.RestartPolicyNever, }, } + e2epod.SetNodeSelection(pod, node) for i, csiVolume := range csiVolumes { name := fmt.Sprintf("my-volume-%d", i) diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 72efe79d296..018bb4a37f5 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -567,9 +567,6 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command }, }, Spec: v1.PodSpec{ - NodeName: node.Name, - NodeSelector: node.Selector, - Affinity: node.Affinity, Containers: []v1.Container{ { Name: "volume-tester", @@ -598,6 +595,7 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command }, } + e2epod.SetNodeSelection(pod, node) pod, err := c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) framework.ExpectNoError(err, "Failed to create pod: %v", err) return pod diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index 7606ac4376e..355b7962bb1 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -149,14 +149,17 @@ func (s *subPathTestSuite) DefineTests(driver TestDriver, pattern testpatterns.T framework.Failf("SubPath test doesn't support: %s", volType) } + nodeSelection := e2epod.NodeSelection{ + Name: l.config.ClientNodeName, + Selector: l.config.ClientNodeSelector, + } + subPath := f.Namespace.Name l.pod = SubpathTestPod(f, subPath, string(volType), l.resource.VolSource, true) - l.pod.Spec.NodeName = l.config.ClientNodeName - l.pod.Spec.NodeSelector = l.config.ClientNodeSelector + e2epod.SetNodeSelection(l.pod, nodeSelection) l.formatPod = volumeFormatPod(f, l.resource.VolSource) - l.formatPod.Spec.NodeName = l.config.ClientNodeName - l.formatPod.Spec.NodeSelector = l.config.ClientNodeSelector + e2epod.SetNodeSelection(l.formatPod, nodeSelection) l.subPathDir = filepath.Join(volumePath, subPath) l.filePathInSubpath = filepath.Join(volumePath, fileName) diff --git a/test/e2e/storage/testsuites/volume_io.go b/test/e2e/storage/testsuites/volume_io.go index 07412bff3de..aea21e4100a 100644 --- a/test/e2e/storage/testsuites/volume_io.go +++ b/test/e2e/storage/testsuites/volume_io.go @@ -183,7 +183,7 @@ func createFileSizes(maxFileSize int64) []int64 { func makePodSpec(config volume.TestConfig, initCmd string, volsrc v1.VolumeSource, podSecContext *v1.PodSecurityContext) *v1.Pod { var gracePeriod int64 = 1 volName := fmt.Sprintf("io-volume-%s", config.Namespace) - return &v1.Pod{ + pod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", APIVersion: "v1", @@ -238,10 +238,14 @@ func makePodSpec(config volume.TestConfig, initCmd string, volsrc v1.VolumeSourc }, }, RestartPolicy: v1.RestartPolicyNever, // want pod to fail if init container fails - NodeName: config.ClientNodeName, - NodeSelector: config.NodeSelector, }, } + + e2epod.SetNodeSelection(pod, e2epod.NodeSelection{ + Name: config.ClientNodeName, + Selector: config.NodeSelector, + }) + return pod } // Write `fsize` bytes to `fpath` in the pod, using dd and the `ddInput` file. diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index 2c6a4633f5e..8aec888813b 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -215,7 +215,7 @@ func (t *volumeModeTestSuite) DefineTests(driver TestDriver, pattern testpattern ginkgo.By("Creating pod") pod := e2epod.MakeSecPod(l.ns.Name, []*v1.PersistentVolumeClaim{l.Pvc}, nil, false, "", false, false, e2epv.SELinuxLabel, nil) // Setting node - pod.Spec.NodeName = l.config.ClientNodeName + e2epod.SetNodeSelection(pod, e2epod.NodeSelection{Name: l.config.ClientNodeName}) pod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}) framework.ExpectNoError(err, "Failed to create pod") defer func() { diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index 9a20f53df38..89579e907c2 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -245,10 +245,12 @@ func testScriptInPod( }, }, RestartPolicy: v1.RestartPolicyNever, - NodeSelector: config.ClientNodeSelector, - NodeName: config.ClientNodeName, }, } + e2epod.SetNodeSelection(pod, e2epod.NodeSelection{ + Name: config.ClientNodeName, + Selector: config.ClientNodeSelector, + }) ginkgo.By(fmt.Sprintf("Creating pod %s", pod.Name)) f.TestContainerOutput("exec-volume-test", pod, 0, []string{fileName})