Merge pull request #62303 from msau42/fix-subpath-reconstruct-test

Automatic merge from submit-queue (batch tested with PRs 62650, 62303, 62545, 62375). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Refactor subpath reconstruction tests to use util test

**What this PR does / why we need it**:
Use common util method for reconstruction tests so that test updates will be reflected in all e2es.  Also increase termination grace period to allow more time for volume manager to cleanup before the tests check for leftover mounts

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #62293

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```
This commit is contained in:
Kubernetes Submit Queue 2018-04-16 13:56:07 -07:00 committed by GitHub
commit 26135c5f4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 41 additions and 74 deletions

View File

@ -70,7 +70,7 @@ var _ = utils.SIGDescribe("GenericPersistentVolume[Disruptive]", func() {
func(t disruptiveTest) { func(t disruptiveTest) {
It(t.testItStmt, func() { It(t.testItStmt, func() {
By("Executing Spec") By("Executing Spec")
t.runTest(c, f, clientPod, pvc) t.runTest(c, f, clientPod)
}) })
}(test) }(test)
} }

View File

@ -31,7 +31,7 @@ import (
"k8s.io/kubernetes/test/e2e/storage/utils" "k8s.io/kubernetes/test/e2e/storage/utils"
) )
type testBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim) type testBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod)
type disruptiveTest struct { type disruptiveTest struct {
testItStmt string testItStmt string
runTest testBody runTest testBody
@ -233,7 +233,7 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() {
func(t disruptiveTest) { func(t disruptiveTest) {
It(t.testItStmt, func() { It(t.testItStmt, func() {
By("Executing Spec") By("Executing Spec")
t.runTest(c, f, clientPod, pvc) t.runTest(c, f, clientPod)
}) })
}(test) }(test)
} }

View File

@ -23,7 +23,6 @@ import (
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/fields"
utilerrors "k8s.io/apimachinery/pkg/util/errors" utilerrors "k8s.io/apimachinery/pkg/util/errors"
@ -553,6 +552,11 @@ func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete
pod.Spec.Containers[1].Image = "busybox" pod.Spec.Containers[1].Image = "busybox"
pod.Spec.Containers[1].Command = []string{"/bin/sh", "-ec", "sleep 100000"} pod.Spec.Containers[1].Command = []string{"/bin/sh", "-ec", "sleep 100000"}
// If grace period is too short, then there is not enough time for the volume
// manager to cleanup the volumes
gracePeriod := int64(30)
pod.Spec.TerminationGracePeriodSeconds = &gracePeriod
By(fmt.Sprintf("Creating pod %s", pod.Name)) By(fmt.Sprintf("Creating pod %s", pod.Name))
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
Expect(err).ToNot(HaveOccurred(), "while creating pod") Expect(err).ToNot(HaveOccurred(), "while creating pod")
@ -563,64 +567,7 @@ func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete
pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{}) pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred(), "while getting pod") Expect(err).ToNot(HaveOccurred(), "while getting pod")
nodeIP, err := framework.GetHostExternalAddress(f.ClientSet, pod) utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true)
Expect(err).NotTo(HaveOccurred(), "while getting node IP")
nodeIP = nodeIP + ":22"
By("Expecting the volume mount to be found.")
result, err := framework.SSH(fmt.Sprintf("mount | grep %s | grep -v volume-subpaths", pod.UID), nodeIP, framework.TestContext.Provider)
framework.LogSSHResult(result)
Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
By("Expecting the subpath volume mount to be found.")
result, err = framework.SSH(fmt.Sprintf("cat /proc/self/mountinfo | grep volume-subpaths | grep %s", pod.UID), nodeIP, framework.TestContext.Provider)
framework.LogSSHResult(result)
Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
By("Stopping the kubelet.")
utils.KubeletCommand(utils.KStop, f.ClientSet, pod)
defer func() {
if err != nil {
utils.KubeletCommand(utils.KStart, f.ClientSet, pod)
}
}()
By(fmt.Sprintf("Deleting Pod %q", pod.Name))
if forceDelete {
err = f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, metav1.NewDeleteOptions(0))
} else {
err = f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{})
}
Expect(err).NotTo(HaveOccurred())
By("Starting the kubelet and waiting for pod to delete.")
utils.KubeletCommand(utils.KStart, f.ClientSet, pod)
err = f.WaitForPodTerminated(pod.Name, "")
if !apierrs.IsNotFound(err) && err != nil {
Expect(err).NotTo(HaveOccurred(), "Expected pod to terminate.")
}
if forceDelete {
// With forceDelete, since pods are immediately deleted from API server, there is no way to be sure when volumes are torn down
// so wait some time to finish
time.Sleep(30 * time.Second)
}
By("Expecting the volume mount not to be found.")
result, err = framework.SSH(fmt.Sprintf("mount | grep %s | grep -v volume-subpaths", pod.UID), nodeIP, framework.TestContext.Provider)
framework.LogSSHResult(result)
Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no mount found).")
framework.Logf("Volume unmounted on node %s", pod.Spec.NodeName)
By("Expecting the subpath volume mount not to be found.")
result, err = framework.SSH(fmt.Sprintf("cat /proc/self/mountinfo | grep volume-subpaths | grep %s", pod.UID), nodeIP, framework.TestContext.Provider)
framework.LogSSHResult(result)
Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no subpath mount found).")
framework.Logf("Subpath volume unmounted on node %s", pod.Spec.NodeName)
} }
func podContainerExec(pod *v1.Pod, containerIndex int, bashExec string) (string, error) { func podContainerExec(pod *v1.Pod, containerIndex int, bashExec string) (string, error) {

View File

@ -138,7 +138,7 @@ func getKubeletMainPid(nodeIP string, sudoPresent bool, systemctlPresent bool) s
} }
// TestKubeletRestartsAndRestoresMount tests that a volume mounted to a pod remains mounted after a kubelet restarts // TestKubeletRestartsAndRestoresMount tests that a volume mounted to a pod remains mounted after a kubelet restarts
func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim) { func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
By("Writing to the volume.") By("Writing to the volume.")
file := "/mnt/_SUCCESS" file := "/mnt/_SUCCESS"
out, err := PodExec(clientPod, fmt.Sprintf("touch %s", file)) out, err := PodExec(clientPod, fmt.Sprintf("touch %s", file))
@ -156,18 +156,26 @@ func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Fra
} }
// TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down. // TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down.
// forceDelete is true indicating whether the pod is forcelly deleted. // forceDelete is true indicating whether the pod is forcefully deleted.
func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, forceDelete bool) { func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool) {
nodeIP, err := framework.GetHostExternalAddress(c, clientPod) nodeIP, err := framework.GetHostExternalAddress(c, clientPod)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
nodeIP = nodeIP + ":22" nodeIP = nodeIP + ":22"
By("Expecting the volume mount to be found.") By("Expecting the volume mount to be found.")
result, err := framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider) result, err := framework.SSH(fmt.Sprintf("mount | grep %s | grep -v volume-subpaths", clientPod.UID), nodeIP, framework.TestContext.Provider)
framework.LogSSHResult(result) framework.LogSSHResult(result)
Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.") Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code)) Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
if checkSubpath {
By("Expecting the volume subpath mount to be found.")
result, err := framework.SSH(fmt.Sprintf("cat /proc/self/mountinfo | grep %s | grep volume-subpaths", clientPod.UID), nodeIP, framework.TestContext.Provider)
framework.LogSSHResult(result)
Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
}
By("Stopping the kubelet.") By("Stopping the kubelet.")
KubeletCommand(KStop, c, clientPod) KubeletCommand(KStop, c, clientPod)
defer func() { defer func() {
@ -182,8 +190,10 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
err = c.CoreV1().Pods(clientPod.Namespace).Delete(clientPod.Name, &metav1.DeleteOptions{}) err = c.CoreV1().Pods(clientPod.Namespace).Delete(clientPod.Name, &metav1.DeleteOptions{})
} }
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// Wait for pod to enter "Terminating state" // Wait for pod to enter "Terminating state"
time.Sleep(30 * time.Second) time.Sleep(30 * time.Second)
By("Starting the kubelet and waiting for pod to delete.") By("Starting the kubelet and waiting for pod to delete.")
KubeletCommand(KStart, c, clientPod) KubeletCommand(KStart, c, clientPod)
err = f.WaitForPodTerminated(clientPod.Name, "") err = f.WaitForPodTerminated(clientPod.Name, "")
@ -196,22 +206,32 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
// so wait some time to finish // so wait some time to finish
time.Sleep(30 * time.Second) time.Sleep(30 * time.Second)
} }
By("Expecting the volume mount not to be found.") By("Expecting the volume mount not to be found.")
result, err = framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider) result, err = framework.SSH(fmt.Sprintf("mount | grep %s | grep -v volume-subpaths", clientPod.UID), nodeIP, framework.TestContext.Provider)
framework.LogSSHResult(result) framework.LogSSHResult(result)
Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.") Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no mount found).") Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no mount found).")
framework.Logf("Volume unmounted on node %s", clientPod.Spec.NodeName) framework.Logf("Volume unmounted on node %s", clientPod.Spec.NodeName)
if checkSubpath {
By("Expecting the volume subpath mount not to be found.")
result, err = framework.SSH(fmt.Sprintf("cat /proc/self/mountinfo | grep %s | grep volume-subpaths", clientPod.UID), nodeIP, framework.TestContext.Provider)
framework.LogSSHResult(result)
Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no subpath mount found).")
framework.Logf("Subpath volume unmounted on node %s", clientPod.Spec.NodeName)
}
} }
// TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down. // TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down.
func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim) { func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, pvc, false) TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false)
} }
// TestVolumeUnmountsFromFoceDeletedPod tests that a volume unmounts if the client pod was forcelly deleted while the kubelet was down. // TestVolumeUnmountsFromFoceDeletedPod tests that a volume unmounts if the client pod was forcefully deleted while the kubelet was down.
func TestVolumeUnmountsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim) { func TestVolumeUnmountsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, pvc, true) TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false)
} }
// RunInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory. // RunInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.

View File

@ -177,7 +177,7 @@ var _ = utils.SIGDescribe("PersistentVolumes:vsphere", func() {
3. Verify that written file is accessible after kubelet restart 3. Verify that written file is accessible after kubelet restart
*/ */
It("should test that a file written to the vspehre volume mount before kubelet restart can be read after restart [Disruptive]", func() { It("should test that a file written to the vspehre volume mount before kubelet restart can be read after restart [Disruptive]", func() {
utils.TestKubeletRestartsAndRestoresMount(c, f, clientPod, pvc) utils.TestKubeletRestartsAndRestoresMount(c, f, clientPod)
}) })
/* /*
@ -192,7 +192,7 @@ var _ = utils.SIGDescribe("PersistentVolumes:vsphere", func() {
5. Verify that volume mount not to be found. 5. Verify that volume mount not to be found.
*/ */
It("should test that a vspehre volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns [Disruptive]", func() { It("should test that a vspehre volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns [Disruptive]", func() {
utils.TestVolumeUnmountsFromDeletedPod(c, f, clientPod, pvc) utils.TestVolumeUnmountsFromDeletedPod(c, f, clientPod)
}) })
/* /*