From c8926bc8e0f289ed4bb6ada30efab28cce79b819 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 19 Mar 2018 13:59:05 -0700 Subject: [PATCH] Add pod deletion to subpath tests, and subpath as file with container restart --- test/e2e/storage/subpath.go | 247 +++++++++++++++++------------------- 1 file changed, 115 insertions(+), 132 deletions(-) diff --git a/test/e2e/storage/subpath.go b/test/e2e/storage/subpath.go index 33e20814009..557769c2670 100644 --- a/test/e2e/storage/subpath.go +++ b/test/e2e/storage/subpath.go @@ -40,11 +40,13 @@ import ( ) var ( - volumePath = "/test-volume" - volumeName = "test-volume" - fileName = "test-file" - retryDuration = 180 - mountImage = imageutils.GetE2EImage(imageutils.Mounttest) + volumePath = "/test-volume" + volumeName = "test-volume" + probeVolumePath = "/probe-volume" + probeFilePath = probeVolumePath + "/probe-file" + fileName = "test-file" + retryDuration = 10 + mountImage = imageutils.GetE2EImage(imageutils.Mounttest) ) type volInfo struct { @@ -88,109 +90,47 @@ var _ = utils.SIGDescribe("Subpath", func() { secret := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "my-secret"}, Data: map[string][]byte{"secret-key": []byte("secret-value")}} secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret) if err != nil && !apierrors.IsAlreadyExists(err) { - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while creating secret") } configmap := &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "my-configmap"}, Data: map[string]string{"configmap-key": "configmap-value"}} configmap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configmap) if err != nil && !apierrors.IsAlreadyExists(err) { - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while creating configmap") } }) - gracePeriod := int64(1) - It("should support subpaths with secret pod", func() { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "pod-subpath-test-secret"}, - Spec: v1.PodSpec{ - Containers: []v1.Container{{ - Name: "test", - Image: mountImage, - Args: []string{"--file_content=" + volumePath}, - VolumeMounts: []v1.VolumeMount{{Name: "secret", MountPath: volumePath, SubPath: "secret-key"}}, - }}, - RestartPolicy: v1.RestartPolicyNever, - TerminationGracePeriodSeconds: &gracePeriod, - Volumes: []v1.Volume{{Name: "secret", VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: "my-secret"}}}}, - }, - } - By(fmt.Sprintf("Creating pod %s", pod.Name)) - f.TestContainerOutput("secret", pod, 0, []string{"secret-value"}) + pod := testPodSubpath(f, "secret-key", "secret", &v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: "my-secret"}}) + testBasicSubpath(f, "secret-value", pod) }) It("should support subpaths with configmap pod", func() { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "pod-subpath-test-configmap"}, - Spec: v1.PodSpec{ - Containers: []v1.Container{{ - Name: "test", - Image: mountImage, - Args: []string{"--file_content=" + volumePath}, - VolumeMounts: []v1.VolumeMount{{Name: "configmap", MountPath: volumePath, SubPath: "configmap-key"}}, - }}, - RestartPolicy: v1.RestartPolicyNever, - TerminationGracePeriodSeconds: &gracePeriod, - Volumes: []v1.Volume{{Name: "configmap", VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{LocalObjectReference: v1.LocalObjectReference{Name: "my-configmap"}}}}}, - }, - } - By(fmt.Sprintf("Creating pod %s", pod.Name)) - f.TestContainerOutput("configmap", pod, 0, []string{"configmap-value"}) + pod := testPodSubpath(f, "configmap-key", "configmap", &v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{LocalObjectReference: v1.LocalObjectReference{Name: "my-configmap"}}}) + testBasicSubpath(f, "configmap-value", pod) }) It("should support subpaths with downward pod", func() { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "pod-subpath-test-downward"}, - Spec: v1.PodSpec{ - Containers: []v1.Container{{ - Name: "test", - Image: mountImage, - Args: []string{"--file_content=" + volumePath}, - VolumeMounts: []v1.VolumeMount{{Name: "downward", MountPath: volumePath, SubPath: "downward/podname"}}, - }}, - RestartPolicy: v1.RestartPolicyNever, - TerminationGracePeriodSeconds: &gracePeriod, - Volumes: []v1.Volume{ - {Name: "downward", VolumeSource: v1.VolumeSource{ - DownwardAPI: &v1.DownwardAPIVolumeSource{ - Items: []v1.DownwardAPIVolumeFile{{Path: "downward/podname", FieldRef: &v1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}, - }, - }}, - }, + pod := testPodSubpath(f, "downward/podname", "downwardAPI", &v1.VolumeSource{ + DownwardAPI: &v1.DownwardAPIVolumeSource{ + Items: []v1.DownwardAPIVolumeFile{{Path: "downward/podname", FieldRef: &v1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}, }, - } - By(fmt.Sprintf("Creating pod %s", pod.Name)) - f.TestContainerOutput("downward", pod, 0, []string{"content of file \"" + volumePath + "\": " + pod.Name}) + }) + testBasicSubpath(f, pod.Name, pod) }) It("should support subpaths with projected pod", func() { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "pod-subpath-test-secret"}, - Spec: v1.PodSpec{ - Containers: []v1.Container{{ - Name: "test", - Image: mountImage, - Args: []string{"--file_content=" + volumePath}, - VolumeMounts: []v1.VolumeMount{{Name: "projected", MountPath: volumePath, SubPath: "projected/configmap-key"}}, - }}, - RestartPolicy: v1.RestartPolicyNever, - TerminationGracePeriodSeconds: &gracePeriod, - Volumes: []v1.Volume{ - {Name: "projected", VolumeSource: v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - {ConfigMap: &v1.ConfigMapProjection{ - LocalObjectReference: v1.LocalObjectReference{Name: "my-configmap"}, - Items: []v1.KeyToPath{{Path: "projected/configmap-key", Key: "configmap-key"}}, - }}, - }, - }, + pod := testPodSubpath(f, "projected/configmap-key", "projected", &v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + {ConfigMap: &v1.ConfigMapProjection{ + LocalObjectReference: v1.LocalObjectReference{Name: "my-configmap"}, + Items: []v1.KeyToPath{{Path: "projected/configmap-key", Key: "configmap-key"}}, }}, }, }, - } - By(fmt.Sprintf("Creating pod %s", pod.Name)) - f.TestContainerOutput("projected", pod, 0, []string{"configmap-value"}) + }) + testBasicSubpath(f, "configmap-value", pod) }) }) @@ -207,15 +147,14 @@ var _ = utils.SIGDescribe("Subpath", func() { filePathInSubpath = filepath.Join(volumePath, fileName) filePathInVolume = filepath.Join(subPathDir, fileName) volInfo := vol.createVolume(f) - pod = testPodSubpath(subPath, curVolType, volInfo.source) - pod.Namespace = f.Namespace.Name + pod = testPodSubpath(f, subPath, curVolType, volInfo.source) pod.Spec.NodeName = volInfo.node }) AfterEach(func() { By("Deleting pod") err := framework.DeletePodWithWait(f, f.ClientSet, pod) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while deleting pod") By("Cleaning up volume") vol.cleanupVolume(f) @@ -248,6 +187,13 @@ var _ = utils.SIGDescribe("Subpath", func() { testReadFile(f, filePathInSubpath, pod, 0) }) + It("should support file as subpath", func() { + // Create the file in the init container + setInitCommand(pod, fmt.Sprintf("echo %s > %s", f.Namespace.Name, subPathDir)) + + testBasicSubpath(f, f.Namespace.Name, pod) + }) + It("should fail if subpath directory is outside the volume [Slow]", func() { // Create the subpath outside the volume setInitCommand(pod, fmt.Sprintf("ln -s /bin %s", subPathDir)) @@ -302,11 +248,18 @@ var _ = utils.SIGDescribe("Subpath", func() { testMultipleReads(f, pod, 0, filepath1, filepath2) }) - It("should support restarting containers [Slow]", func() { + It("should support restarting containers using directory as subpath [Slow]", func() { // Create the directory - setInitCommand(pod, fmt.Sprintf("mkdir -p %v", subPathDir)) + setInitCommand(pod, fmt.Sprintf("mkdir -p %v; touch %v", subPathDir, probeFilePath)) - testPodContainerRestart(f, pod, filePathInVolume, filePathInSubpath) + testPodContainerRestart(f, pod) + }) + + It("should support restarting containers using file as subpath [Slow]", func() { + // Create the file + setInitCommand(pod, fmt.Sprintf("touch %v; touch %v", subPathDir, probeFilePath)) + + testPodContainerRestart(f, pod) }) It("should unmount if pod is gracefully deleted while kubelet is down [Disruptive][Slow]", func() { @@ -325,13 +278,28 @@ var _ = utils.SIGDescribe("Subpath", func() { // TODO: add a test case for the same disk with two partitions }) -func testPodSubpath(subpath, volumeType string, source *v1.VolumeSource) *v1.Pod { - suffix := strings.ToLower(fmt.Sprintf("%s-%s", volumeType, rand.String(4))) - privileged := true - gracePeriod := int64(1) +func testBasicSubpath(f *framework.Framework, contents string, pod *v1.Pod) { + setReadCommand(volumePath, &pod.Spec.Containers[0]) + + By(fmt.Sprintf("Creating pod %s", pod.Name)) + f.TestContainerOutput("atomic-volume-subpath", pod, 0, []string{contents}) + + By(fmt.Sprintf("Deleting pod %s", pod.Name)) + err := framework.DeletePodWithWait(f, f.ClientSet, pod) + Expect(err).NotTo(HaveOccurred(), "while deleting pod") +} + +func testPodSubpath(f *framework.Framework, subpath, volumeType string, source *v1.VolumeSource) *v1.Pod { + var ( + suffix = strings.ToLower(fmt.Sprintf("%s-%s", volumeType, rand.String(4))) + privileged = true + gracePeriod = int64(1) + probeVolumeName = "liveness-probe-volume" + ) return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("pod-subpath-test-%s", suffix), + Name: fmt.Sprintf("pod-subpath-test-%s", suffix), + Namespace: f.Namespace.Name, }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ @@ -343,6 +311,10 @@ func testPodSubpath(subpath, volumeType string, source *v1.VolumeSource) *v1.Pod Name: volumeName, MountPath: volumePath, }, + { + Name: probeVolumeName, + MountPath: probeVolumePath, + }, }, SecurityContext: &v1.SecurityContext{ Privileged: &privileged, @@ -359,6 +331,10 @@ func testPodSubpath(subpath, volumeType string, source *v1.VolumeSource) *v1.Pod MountPath: volumePath, SubPath: subpath, }, + { + Name: probeVolumeName, + MountPath: probeVolumePath, + }, }, SecurityContext: &v1.SecurityContext{ Privileged: &privileged, @@ -372,6 +348,10 @@ func testPodSubpath(subpath, volumeType string, source *v1.VolumeSource) *v1.Pod Name: volumeName, MountPath: volumePath, }, + { + Name: probeVolumeName, + MountPath: probeVolumePath, + }, }, SecurityContext: &v1.SecurityContext{ Privileged: &privileged, @@ -385,6 +365,12 @@ func testPodSubpath(subpath, volumeType string, source *v1.VolumeSource) *v1.Pod Name: volumeName, VolumeSource: *source, }, + { + Name: probeVolumeName, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, }, }, } @@ -435,17 +421,21 @@ func testReadFile(f *framework.Framework, file string, pod *v1.Pod, containerInd f.TestContainerOutput("subpath", pod, containerIndex, []string{ "content of file \"" + file + "\": mount-tester new file", }) + + By(fmt.Sprintf("Deleting pod %s", pod.Name)) + err := framework.DeletePodWithWait(f, f.ClientSet, pod) + Expect(err).NotTo(HaveOccurred(), "while deleting pod") } func testPodFailSupath(f *framework.Framework, pod *v1.Pod) { By(fmt.Sprintf("Creating pod %s", pod.Name)) pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while creating pod") defer func() { framework.DeletePodWithWait(f, f.ClientSet, pod) }() err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, time.Minute) - Expect(err).To(HaveOccurred()) + Expect(err).To(HaveOccurred(), "while waiting for pod to be running") By("Checking for subpath error event") selector := fields.Set{ @@ -456,45 +446,44 @@ func testPodFailSupath(f *framework.Framework, pod *v1.Pod) { }.AsSelector().String() options := metav1.ListOptions{FieldSelector: selector} events, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(options) - Expect(err).NotTo(HaveOccurred()) - Expect(len(events.Items)).NotTo(Equal(0)) - Expect(events.Items[0].Message).To(ContainSubstring("subPath")) + Expect(err).NotTo(HaveOccurred(), "while getting pod events") + Expect(len(events.Items)).NotTo(Equal(0), "no events found") + Expect(events.Items[0].Message).To(ContainSubstring("subPath"), "subpath error not found") } -func testPodContainerRestart(f *framework.Framework, pod *v1.Pod, fileInVolume, fileInSubpath string) { +// Tests that the existing subpath mount is detected when a container restarts +func testPodContainerRestart(f *framework.Framework, pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure - // Add liveness probe to subpath container pod.Spec.Containers[0].Image = "busybox" pod.Spec.Containers[0].Command = []string{"/bin/sh", "-ec", "sleep 100000"} + pod.Spec.Containers[1].Image = "busybox" + pod.Spec.Containers[1].Command = []string{"/bin/sh", "-ec", "sleep 100000"} + + // Add liveness probe to subpath container pod.Spec.Containers[0].LivenessProbe = &v1.Probe{ Handler: v1.Handler{ Exec: &v1.ExecAction{ - Command: []string{"cat", fileInSubpath}, + Command: []string{"cat", probeFilePath}, }, }, - InitialDelaySeconds: 15, + InitialDelaySeconds: 1, FailureThreshold: 1, PeriodSeconds: 2, } - // Set volume container to write file - writeCmd := fmt.Sprintf("echo test > %v", fileInVolume) - pod.Spec.Containers[1].Image = "busybox" - pod.Spec.Containers[1].Command = []string{"/bin/sh", "-ec", fmt.Sprintf("%v; sleep 100000", writeCmd)} - // Start pod By(fmt.Sprintf("Creating pod %s", pod.Name)) pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while creating pod") err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, time.Minute) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while waiting for pod to be running") By("Failing liveness probe") - out, err := podContainerExec(pod, 1, fmt.Sprintf("rm %v", fileInVolume)) + out, err := podContainerExec(pod, 1, fmt.Sprintf("rm %v", probeFilePath)) framework.Logf("Pod exec output: %v", out) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while failing liveness probe") // Check that container has restarted By("Waiting for container to restart") @@ -516,14 +505,14 @@ func testPodContainerRestart(f *framework.Framework, pod *v1.Pod, fileInVolume, } return false, nil }) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while waiting for container to restart") // Fix liveness probe By("Rewriting the file") - writeCmd = fmt.Sprintf("echo test-after > %v", fileInVolume) + writeCmd := fmt.Sprintf("echo test-after > %v", probeFilePath) out, err = podContainerExec(pod, 1, writeCmd) framework.Logf("Pod exec output: %v", out) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while rewriting the probe file") // Wait for container restarts to stabilize By("Waiting for container to stop restarting") @@ -552,13 +541,7 @@ func testPodContainerRestart(f *framework.Framework, pod *v1.Pod, fileInVolume, } return false, nil }) - Expect(err).ToNot(HaveOccurred()) - - // Verify content of file in subpath - out, err = podContainerExec(pod, 0, fmt.Sprintf("cat %v", fileInSubpath)) - framework.Logf("Pod exec output: %v", out) - Expect(err).ToNot(HaveOccurred()) - Expect(strings.TrimSpace(out)).To(BeEquivalentTo("test-after")) + Expect(err).ToNot(HaveOccurred(), "while waiting for container to stabilize") } func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete bool) { @@ -572,16 +555,16 @@ func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete By(fmt.Sprintf("Creating pod %s", pod.Name)) pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while creating pod") err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, time.Minute) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while waiting for pod to be running") pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while getting pod") nodeIP, err := framework.GetHostExternalAddress(f.ClientSet, pod) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), "while getting node IP") nodeIP = nodeIP + ":22" By("Expecting the volume mount to be found.") @@ -718,13 +701,13 @@ func (s *hostpathSymlinkSource) createVolume(f *framework.Framework) volInfo { }, } pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while creating hostpath init pod") err = framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, pod.Namespace) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while waiting for hostpath init pod to succeed") err = framework.DeletePodWithWait(f, f.ClientSet, pod) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), "while deleting hostpath init pod") return volInfo{ source: &v1.VolumeSource{