From 707883d52b9d89496d2c62ee383e30875b1bc762 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Sun, 26 Apr 2020 00:26:44 -0700 Subject: [PATCH 1/2] tests: Refactors Variable Expansion tests --- test/e2e/common/expansion.go | 553 ++++++++++++++--------------------- 1 file changed, 215 insertions(+), 338 deletions(-) diff --git a/test/e2e/common/expansion.go b/test/e2e/common/expansion.go index aa2f7b77267..eb0f7321fe9 100644 --- a/test/e2e/common/expansion.go +++ b/test/e2e/common/expansion.go @@ -44,37 +44,21 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { Description: Create a Pod with environment variables. Environment variables defined using previously defined environment variables MUST expand to proper values. */ framework.ConformanceIt("should allow composing env vars into new env vars [NodeConformance]", func() { - podName := "var-expansion-" + string(uuid.NewUUID()) - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, + envVars := []v1.EnvVar{ + { + Name: "FOO", + Value: "foo-value", }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dapi-container", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"sh", "-c", "env"}, - Env: []v1.EnvVar{ - { - Name: "FOO", - Value: "foo-value", - }, - { - Name: "BAR", - Value: "bar-value", - }, - { - Name: "FOOBAR", - Value: "$(FOO);;$(BAR)", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, + { + Name: "BAR", + Value: "bar-value", + }, + { + Name: "FOOBAR", + Value: "$(FOO);;$(BAR)", }, } + pod := newPod([]string{"sh", "-c", "env"}, envVars, nil, nil) f.TestContainerOutput("env composition", pod, 0, []string{ "FOO=foo-value", @@ -89,29 +73,13 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { Description: Create a Pod with environment variables and container command using them. Container command using the defined environment variables MUST expand to proper values. */ framework.ConformanceIt("should allow substituting values in a container's command [NodeConformance]", func() { - podName := "var-expansion-" + string(uuid.NewUUID()) - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dapi-container", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"sh", "-c", "TEST_VAR=wrong echo \"$(TEST_VAR)\""}, - Env: []v1.EnvVar{ - { - Name: "TEST_VAR", - Value: "test-value", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, + envVars := []v1.EnvVar{ + { + Name: "TEST_VAR", + Value: "test-value", }, } + pod := newPod([]string{"sh", "-c", "TEST_VAR=wrong echo \"$(TEST_VAR)\""}, envVars, nil, nil) f.TestContainerOutput("substitution in container's command", pod, 0, []string{ "test-value", @@ -124,30 +92,14 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { Description: Create a Pod with environment variables and container command arguments using them. Container command arguments using the defined environment variables MUST expand to proper values. */ framework.ConformanceIt("should allow substituting values in a container's args [NodeConformance]", func() { - podName := "var-expansion-" + string(uuid.NewUUID()) - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dapi-container", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"sh", "-c"}, - Args: []string{"TEST_VAR=wrong echo \"$(TEST_VAR)\""}, - Env: []v1.EnvVar{ - { - Name: "TEST_VAR", - Value: "test-value", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, + envVars := []v1.EnvVar{ + { + Name: "TEST_VAR", + Value: "test-value", }, } + pod := newPod([]string{"sh", "-c"}, envVars, nil, nil) + pod.Spec.Containers[0].Args = []string{"TEST_VAR=wrong echo \"$(TEST_VAR)\""} f.TestContainerOutput("substitution in container's args", pod, 0, []string{ "test-value", @@ -160,48 +112,34 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { Description: Make sure a container's subpath can be set using an expansion of environment variables. */ framework.ConformanceIt("should allow substituting values in a volume subpath [sig-storage]", func() { - podName := "var-expansion-" + string(uuid.NewUUID()) - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, + envVars := []v1.EnvVar{ + { + Name: "POD_NAME", + Value: "foo", }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dapi-container", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"sh", "-c", "test -d /testcontainer/" + podName + ";echo $?"}, - Env: []v1.EnvVar{ - { - Name: "POD_NAME", - Value: podName, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "workdir1", - MountPath: "/logscontainer", - SubPathExpr: "$(POD_NAME)", - }, - { - Name: "workdir1", - MountPath: "/testcontainer", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "workdir1", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, + } + mounts := []v1.VolumeMount{ + { + Name: "workdir1", + MountPath: "/logscontainer", + SubPathExpr: "$(POD_NAME)", + }, + { + Name: "workdir1", + MountPath: "/testcontainer", + }, + } + volumes := []v1.Volume{ + { + Name: "workdir1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, }, }, } + pod := newPod([]string{}, envVars, mounts, volumes) + envVars[0].Value = pod.ObjectMeta.Name + pod.Spec.Containers[0].Command = []string{"sh", "-c", "test -d /testcontainer/" + pod.ObjectMeta.Name + ";echo $?"} f.TestContainerOutput("substitution in volume subpath", pod, 0, []string{ "0", @@ -215,43 +153,28 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { */ framework.ConformanceIt("should fail substituting values in a volume subpath with backticks [sig-storage][Slow]", func() { - podName := "var-expansion-" + string(uuid.NewUUID()) - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, + envVars := []v1.EnvVar{ + { + Name: "POD_NAME", + Value: "..", }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dapi-container", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Env: []v1.EnvVar{ - { - Name: "POD_NAME", - Value: "..", - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "workdir1", - MountPath: "/logscontainer", - SubPathExpr: "$(POD_NAME)", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "workdir1", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, + } + mounts := []v1.VolumeMount{ + { + Name: "workdir1", + MountPath: "/logscontainer", + SubPathExpr: "$(POD_NAME)", + }, + } + volumes := []v1.Volume{ + { + Name: "workdir1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, }, }, } + pod := newPod(nil, envVars, mounts, volumes) // Pod should fail testPodFailSubpath(f, pod) @@ -264,43 +187,28 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { */ framework.ConformanceIt("should fail substituting values in a volume subpath with absolute path [sig-storage][Slow]", func() { - podName := "var-expansion-" + string(uuid.NewUUID()) - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, + envVars := []v1.EnvVar{ + { + Name: "POD_NAME", + Value: "/tmp", }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "dapi-container", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Env: []v1.EnvVar{ - { - Name: "POD_NAME", - Value: "/tmp", - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "workdir1", - MountPath: "/logscontainer", - SubPathExpr: "$(POD_NAME)", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "workdir1", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, + } + mounts := []v1.VolumeMount{ + { + Name: "workdir1", + MountPath: "/logscontainer", + SubPathExpr: "$(POD_NAME)", + }, + } + volumes := []v1.Volume{ + { + Name: "workdir1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, }, }, } + pod := newPod(nil, envVars, mounts, volumes) // Pod should fail testPodFailSubpath(f, pod) @@ -313,58 +221,42 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { */ framework.ConformanceIt("should verify that a failing subpath expansion can be modified during the lifecycle of a container [sig-storage][Slow]", func() { - podName := "var-expansion-" + string(uuid.NewUUID()) - containerName := "dapi-container" - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, - Annotations: map[string]string{"notmysubpath": "mypath"}, + envVars := []v1.EnvVar{ + { + Name: "POD_NAME", + Value: "foo", }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: containerName, - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"sh", "-c", "tail -f /dev/null"}, - Env: []v1.EnvVar{ - { - Name: "POD_NAME", - Value: "foo", - }, - { - Name: "ANNOTATION", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.annotations['mysubpath']", - }, - }, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "workdir1", - MountPath: "/subpath_mount", - SubPathExpr: "$(ANNOTATION)/$(POD_NAME)", - }, - { - Name: "workdir1", - MountPath: "/volume_mount", - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "workdir1", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, + { + Name: "ANNOTATION", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.annotations['mysubpath']", }, }, }, } + mounts := []v1.VolumeMount{ + { + Name: "workdir1", + MountPath: "/subpath_mount", + SubPathExpr: "$(ANNOTATION)/$(POD_NAME)", + }, + { + Name: "workdir1", + MountPath: "/volume_mount", + }, + } + volumes := []v1.Volume{ + { + Name: "workdir1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + pod := newPod([]string{"sh", "-c", "tail -f /dev/null"}, envVars, mounts, volumes) + pod.ObjectMeta.Annotations = map[string]string{"notmysubpath": "mypath"} ginkgo.By("creating the pod with failed condition") var podClient *framework.PodClient = f.PodClient() @@ -374,7 +266,7 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { framework.ExpectError(err, "while waiting for pod to be running") ginkgo.By("updating the pod") - podClient.Update(podName, func(pod *v1.Pod) { + podClient.Update(pod.ObjectMeta.Name, func(pod *v1.Pod) { pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "mypath"} }) @@ -398,59 +290,42 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { */ framework.ConformanceIt("should succeed in writing subpaths in container [sig-storage][Slow]", func() { - podName := "var-expansion-" + string(uuid.NewUUID()) - containerName := "dapi-container" - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, - Annotations: map[string]string{"mysubpath": "mypath"}, + envVars := []v1.EnvVar{ + { + Name: "POD_NAME", + Value: "foo", }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: containerName, - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"sh", "-c", "tail -f /dev/null"}, - Env: []v1.EnvVar{ - { - Name: "POD_NAME", - Value: "foo", - }, - { - Name: "ANNOTATION", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.annotations['mysubpath']", - }, - }, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "workdir1", - MountPath: "/subpath_mount", - SubPathExpr: "$(ANNOTATION)/$(POD_NAME)", - }, - { - Name: "workdir1", - MountPath: "/volume_mount", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "workdir1", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, + { + Name: "ANNOTATION", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.annotations['mysubpath']", }, }, }, } + mounts := []v1.VolumeMount{ + { + Name: "workdir1", + MountPath: "/subpath_mount", + SubPathExpr: "$(ANNOTATION)/$(POD_NAME)", + }, + { + Name: "workdir1", + MountPath: "/volume_mount", + }, + } + volumes := []v1.Volume{ + { + Name: "workdir1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + pod := newPod([]string{"sh", "-c", "tail -f /dev/null"}, envVars, mounts, volumes) + pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "mypath"} ginkgo.By("creating the pod") var podClient *framework.PodClient = f.PodClient() @@ -475,7 +350,7 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { } ginkgo.By("updating the annotation value") - podClient.Update(podName, func(pod *v1.Pod) { + podClient.Update(pod.ObjectMeta.Name, func(pod *v1.Pod) { pod.ObjectMeta.Annotations["mysubpath"] = "mynewpath" }) @@ -499,75 +374,52 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { */ framework.ConformanceIt("should not change the subpath mount on a container restart if the environment variable changes [sig-storage][Slow]", func() { - - suffix := string(uuid.NewUUID()) - podName := fmt.Sprintf("var-expansion-%s", suffix) - containerName := "dapi-container" - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"name": podName}, - Annotations: map[string]string{"mysubpath": "foo"}, - }, - Spec: v1.PodSpec{ - InitContainers: []v1.Container{ - { - Name: fmt.Sprintf("init-volume-%s", suffix), - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"sh", "-c", "mkdir -p /volume_mount/foo; touch /volume_mount/foo/test.log"}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "workdir1", - MountPath: "/subpath_mount", - }, - { - Name: "workdir1", - MountPath: "/volume_mount", - }, - }, - }, - }, - - Containers: []v1.Container{ - { - Name: containerName, - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"/bin/sh", "-ec", "sleep 100000"}, - Env: []v1.EnvVar{ - { - Name: "POD_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.annotations['mysubpath']", - }, - }, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "workdir1", - MountPath: "/subpath_mount", - SubPathExpr: "$(POD_NAME)", - }, - { - Name: "workdir1", - MountPath: "/volume_mount", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyOnFailure, - Volumes: []v1.Volume{ - { - Name: "workdir1", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, + envVars := []v1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.annotations['mysubpath']", }, }, }, } + mounts := []v1.VolumeMount{ + { + Name: "workdir1", + MountPath: "/subpath_mount", + }, + { + Name: "workdir1", + MountPath: "/volume_mount", + }, + } + subpathMounts := []v1.VolumeMount{ + { + Name: "workdir1", + MountPath: "/subpath_mount", + SubPathExpr: "$(POD_NAME)", + }, + { + Name: "workdir1", + MountPath: "/volume_mount", + }, + } + volumes := []v1.Volume{ + { + Name: "workdir1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + pod := newPod([]string{"/bin/sh", "-ec", "sleep 100000"}, envVars, subpathMounts, volumes) + pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure + pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "foo"} + suffix := string(uuid.NewUUID()) + pod.Spec.InitContainers = []v1.Container{newContainer( + fmt.Sprintf("init-volume-%s", suffix), []string{"sh", "-c", "mkdir -p /volume_mount/foo; touch /volume_mount/foo/test.log"}, nil, mounts)} // Add liveness probe to subpath container pod.Spec.Containers[0].LivenessProbe = &v1.Probe{ @@ -593,7 +445,7 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { framework.ExpectNoError(err, "while waiting for pod to be running") ginkgo.By("updating the pod") - podClient.Update(podName, func(pod *v1.Pod) { + podClient.Update(pod.ObjectMeta.Name, func(pod *v1.Pod) { pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "newsubpath"} }) @@ -693,3 +545,28 @@ func waitForPodContainerRestart(f *framework.Framework, pod *v1.Pod, volumeMount }) framework.ExpectNoError(err, "while waiting for container to stabilize") } + +func newPod(command []string, envVars []v1.EnvVar, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { + podName := "var-expansion-" + string(uuid.NewUUID()) + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Labels: map[string]string{"name": podName}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{newContainer("dapi-container", command, envVars, mounts)}, + RestartPolicy: v1.RestartPolicyNever, + Volumes: volumes, + }, + } +} + +func newContainer(containerName string, command []string, envVars []v1.EnvVar, mounts []v1.VolumeMount) v1.Container { + return v1.Container{ + Name: containerName, + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: command, + Env: envVars, + VolumeMounts: mounts, + } +} From 06a1b7b4911ea9bc5a7eea71952356a5a577bf3b Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 13 Apr 2020 20:37:51 -0700 Subject: [PATCH 2/2] tests: Retries exec after container restart due to liveness The test "should not change the subpath mount on a container restart if the environment variable changes" creates a pod with the liveness probe: cat /volume_mount/test.log. The test then deletes that file, which causes the probe to fail and the container to be restarted. After which it recreates the file by exec-ing into the pod, but there is a chance that the container was not created yet, or it did not start yet. This commit adds a few retries to the exec command. --- test/e2e/common/expansion.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/common/expansion.go b/test/e2e/common/expansion.go index eb0f7321fe9..f6982f63dae 100644 --- a/test/e2e/common/expansion.go +++ b/test/e2e/common/expansion.go @@ -417,6 +417,8 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { pod := newPod([]string{"/bin/sh", "-ec", "sleep 100000"}, envVars, subpathMounts, volumes) pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "foo"} + sideContainerName := "side-container" + pod.Spec.Containers = append(pod.Spec.Containers, newContainer(sideContainerName, []string{"/bin/sh", "-ec", "sleep 100000"}, envVars, subpathMounts)) suffix := string(uuid.NewUUID()) pod.Spec.InitContainers = []v1.Container{newContainer( fmt.Sprintf("init-volume-%s", suffix), []string{"sh", "-c", "mkdir -p /volume_mount/foo; touch /volume_mount/foo/test.log"}, nil, mounts)} @@ -512,9 +514,8 @@ func waitForPodContainerRestart(f *framework.Framework, pod *v1.Pod, volumeMount // Fix liveness probe ginkgo.By("Rewriting the file") - stdout, _, err = f.ExecShellInPodWithFullOutput(pod.Name, fmt.Sprintf("echo test-after > %v", volumeMount)) + stdout = f.ExecShellInContainer(pod.Name, pod.Spec.Containers[1].Name, fmt.Sprintf("echo test-after > %v", volumeMount)) framework.Logf("Pod exec output: %v", stdout) - framework.ExpectNoError(err, "while rewriting the probe file") // Wait for container restarts to stabilize ginkgo.By("Waiting for container to stop restarting")