diff --git a/test/e2e_node/container_lifecycle_test.go b/test/e2e_node/container_lifecycle_test.go index 8134092ac7e..c3bca649a9c 100644 --- a/test/e2e_node/container_lifecycle_test.go +++ b/test/e2e_node/container_lifecycle_test.go @@ -35,6 +35,8 @@ const ( PostStartPrefix = "PostStart" ) +var containerRestartPolicyAlways = v1.ContainerRestartPolicyAlways + func prefixedName(namePrefix string, name string) string { return fmt.Sprintf("%s-%s", namePrefix, name) } @@ -383,7 +385,7 @@ var _ = SIGDescribe("[NodeConformance] Containers Lifecycle ", func() { podSpec := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "sidecar-runs-with-pod", + Name: "bad-image", }, Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyNever, @@ -417,7 +419,6 @@ var _ = SIGDescribe("[NodeConformance] Containers Lifecycle ", func() { client := e2epod.NewPodClient(f) podSpec = client.Create(context.TODO(), podSpec) - // sidecar should be in image pull backoff err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart) framework.ExpectNoError(err) @@ -596,3 +597,1498 @@ var _ = SIGDescribe("[NodeConformance] Containers Lifecycle ", func() { }) }) }) + +var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle ", func() { + f := framework.NewDefaultFramework("containers-lifecycle-test") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + ginkgo.When("using a Pod with restartPolicy=Never, three init container and two restartable init containers", ginkgo.Ordered, func() { + + init1 := "init-1" + restartableInit1 := "restartable-init-1" + init2 := "init-2" + restartableInit2 := "restartable-init-2" + init3 := "init-3" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-containers-start-serially", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init2, + Image: busyboxImage, + Command: ExecCommand(init2, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + { + Name: restartableInit2, + Image: busyboxImage, + Command: ExecCommand(restartableInit2, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init3, + Image: busyboxImage, + Command: ExecCommand(init3, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should finish and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + // TODO: check for Pod to be succeeded + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + + ginkgo.It("should run the first init container to completion before starting first restartable init container", func() { + framework.ExpectNoError(results.StartsBefore(init1, restartableInit1)) + framework.ExpectNoError(results.ExitsBefore(init1, restartableInit1)) + }) + + ginkgo.It("should start first restartable init container before starting second init container", func() { + framework.ExpectNoError(results.StartsBefore(restartableInit1, init2)) + }) + + ginkgo.It("should run first init container and first restartable init container together", func() { + framework.ExpectNoError(results.RunTogether(restartableInit1, init2)) + }) + + ginkgo.It("should run second init container to completion before starting second restartable init container", func() { + framework.ExpectNoError(results.StartsBefore(init2, restartableInit2)) + framework.ExpectNoError(results.ExitsBefore(init2, restartableInit2)) + }) + + ginkgo.It("should start second restartable init container before third init container", func() { + framework.ExpectNoError(results.StartsBefore(restartableInit2, init3)) + }) + + ginkgo.It("should run both restartable init cotnainers and third init container together", func() { + framework.ExpectNoError(results.RunTogether(restartableInit2, restartableInit1)) + framework.ExpectNoError(results.RunTogether(restartableInit1, init3)) + framework.ExpectNoError(results.RunTogether(restartableInit2, init3)) + }) + + ginkgo.It("should run third init container to completion before starting regular container", func() { + framework.ExpectNoError(results.StartsBefore(init3, regular1)) + framework.ExpectNoError(results.ExitsBefore(init3, regular1)) + }) + + ginkgo.It("should run both restartable init cotnainers and a regular container together", func() { + framework.ExpectNoError(results.RunTogether(restartableInit1, regular1)) + framework.ExpectNoError(results.RunTogether(restartableInit2, regular1)) + }) + }) + + ginkgo.When("using a restartable init container in a Pod with restartPolicy=Never", func() { + ginkgo.When("a restartable init container runs continuously", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-run-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should complete a Pod successfully and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should not restart a restartable init container", func() { + framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1)) + }) + ginkgo.It("should run a regular container to completion", func() { + framework.ExpectNoError(results.Exits(regular1)) + }) + }) + + ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-runs-with-pod", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage), + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should mark a Pod as failed and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + // restartable init container should be in image pull backoff + err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart) + framework.ExpectNoError(err) + + podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should not start a restartable init container", func() { + framework.ExpectNoError(results.DoesntStart(restartableInit1)) + }) + ginkgo.It("should not start a regular container", func() { + framework.ExpectNoError(results.DoesntStart(regular1)) + }) + }) + + // TODO: add a test case similar to one above, but with startup probe never succeeding + + ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + init1 := "init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-exit-0-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 5, + ExitCode: 0, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 60, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should complete a Pod successfully and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should restart a restartable init container before the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) + }) + ginkgo.It("should restart a restartable init container after the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) + }) + ginkgo.It("should run a regular container to completion", func() { + framework.ExpectNoError(results.Exits(regular1)) + }) + }) + + ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() { + restartableInit1 := "restartable-init-1" + init1 := "init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-exit-1-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 5, + ExitCode: 0, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 60, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should complete a Pod successfully and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should restart a restartable init container before the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) + }) + ginkgo.It("should restart a restartable init container after the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) + }) + ginkgo.It("should run a regular container to completion", func() { + framework.ExpectNoError(results.Exits(regular1)) + }) + }) + + ginkgo.When("an Init container before restartable init container fails", ginkgo.Ordered, func() { + + init1 := "init-1" + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "init-container-fails-before-restartable-init-starts", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + }, + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should mark a Pod as failed and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should mark an Init container as failed", func() { + framework.ExpectNoError(results.Exits(init1)) + }) + ginkgo.It("should not start restartable init container", func() { + framework.ExpectNoError(results.DoesntStart(restartableInit1)) + }) + }) + + ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() { + + init1 := "init-1" + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-fails-before-init-container", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should mark a Pod as failed and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitForPodFailedReason(context.TODO(), f.ClientSet, podSpec, "", 1*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should mark an Init container as failed", func() { + framework.ExpectNoError(results.Exits(init1)) + }) + // TODO: how will we be able to test it if restartable init container + // will never fail and there will be no termination log? Or will be? + ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { + framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) + }) + // TODO: check preStop hooks when they are enabled + }) + }) + + ginkgo.When("using a restartable init container in a Pod with restartPolicy=OnFailure", ginkgo.Ordered, func() { + // this test case the same as for restartPolicy=Never + ginkgo.When("a restartable init container runs continuously", func() { + + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-run-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyOnFailure, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should complete a Pod successfully and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should not restart a restartable init container", func() { + framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1)) + }) + ginkgo.It("should run a regular container to completion", func() { + framework.ExpectNoError(results.Exits(regular1)) + }) + }) + + ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-runs-with-pod", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyOnFailure, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage), + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should mark a Pod as failed and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + // restartable init container should be in image pull backoff + err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart) + framework.ExpectNoError(err) + + podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should not start a restartable init container", func() { + framework.ExpectNoError(results.DoesntStart(restartableInit1)) + }) + ginkgo.It("should not start a regular container", func() { + framework.ExpectNoError(results.DoesntStart(regular1)) + }) + }) + + // TODO: add a test case similar to one above, but with startup probe never succeeding + + // this test case the same as for restartPolicy=Never + ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + init1 := "init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-exit-0-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyOnFailure, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 5, + ExitCode: 0, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 60, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should complete a Pod successfully and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should restart a restartable init container before the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) + }) + ginkgo.It("should restart a restartable init container after the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) + }) + ginkgo.It("should run a regular container to completion", func() { + framework.ExpectNoError(results.Exits(regular1)) + }) + }) + + // this test case the same as for restartPolicy=Never + ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + init1 := "init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-exit-1-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyOnFailure, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 5, + ExitCode: 0, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 60, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should complete a Pod successfully and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace, 5*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should restart a restartable init container before the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) + }) + ginkgo.It("should restart a restartable init container after the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) + }) + ginkgo.It("should run a regular container to completion", func() { + framework.ExpectNoError(results.Exits(regular1)) + }) + }) + + ginkgo.When("an Init container before restartable init container continuously fails", ginkgo.Ordered, func() { + + init1 := "init-1" + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "init-container-fails-before-restartable-init-starts", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyOnFailure, + InitContainers: []v1.Container{ + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + }, + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should continuously run Pod keeping it Pending", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) { + if pod.Status.Phase != v1.PodPending { + return false, fmt.Errorf("pod should be in pending phase") + } + if len(pod.Status.InitContainerStatuses) < 1 { + return false, nil + } + containerStatus := pod.Status.InitContainerStatuses[0] + return containerStatus.RestartCount >= 3, nil + }) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should have Init container restartCount greater than 0", func() { + framework.ExpectNoError(results.HasRestarted(init1)) + }) + ginkgo.It("should not start restartable init container", func() { + framework.ExpectNoError(results.DoesntStart(restartableInit1)) + }) + }) + + ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() { + + init1 := "init-1" + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-fails-before-init-container", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyOnFailure, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should continuously run Pod keeping it Pending", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) { + if pod.Status.Phase != v1.PodPending { + return false, fmt.Errorf("pod should be in pending phase") + } + if len(pod.Status.InitContainerStatuses) < 1 { + return false, nil + } + containerStatus := pod.Status.InitContainerStatuses[0] + return containerStatus.RestartCount >= 3, nil + }) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should have Init container restartCount greater than 0", func() { + framework.ExpectNoError(results.HasRestarted(init1)) + }) + // TODO: how will we be able to test it if restartable init container will never fail and there will be no termination log? Or will be? + ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { + framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) + }) + // TODO: check preStop hooks when they are enabled + }) + }) + + ginkgo.When("using a restartable init container in a Pod with restartPolicy=Always", ginkgo.Ordered, func() { + ginkgo.When("a restartable init container runs continuously", func() { + + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-run-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + // regular container should exit at least once so we can get it's termination log + // this test case is different from restartPolicy=Never + ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */ + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 2, 2*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + + ginkgo.It("should not restart a restartable init container", func() { + framework.ExpectNoError(results.DoesntStartAfter(restartableInit1, regular1)) + }) + // this test case is different from restartPolicy=Never + ginkgo.It("should start a regular container", func() { + framework.ExpectNoError(results.HasRestarted(regular1)) + }) + }) + + ginkgo.When("a restartable init container fails to start because of a bad image", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-runs-with-pod", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: imageutils.GetE2EImage(imageutils.InvalidRegistryImage), + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should continuously run Pod keeping it Pending and produce log", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + // restartable init container should be in image pull backoff + err := WaitForPodInitContainerToFail(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, "ImagePullBackOff", f.Timeouts.PodStart) + framework.ExpectNoError(err) + + podSpec, err = client.Get(context.Background(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should not start a restartable init container", func() { + framework.ExpectNoError(results.DoesntStart(restartableInit1)) + }) + ginkgo.It("should not start a regular container", func() { + framework.ExpectNoError(results.DoesntStart(regular1)) + }) + }) + + // TODO: add a test case similar to one above, but with startup probe never succeeding + + ginkgo.When("a restartable init container starts and exits with exit code 0 continuously", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + init1 := "init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-exit-0-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 5, + ExitCode: 0, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 60, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */ + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 1, 2*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should restart a restartable init container before the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) + }) + ginkgo.It("should restart a restartable init container after the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) + }) + ginkgo.It("should start a regular container", func() { + framework.ExpectNoError(results.Starts(regular1)) + }) + }) + + // this test case the same as for restartPolicy=Never + ginkgo.When("a restartable init container starts and exits with exit code 1 continuously", ginkgo.Ordered, func() { + + restartableInit1 := "restartable-init-1" + init1 := "init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-exit-1-continuously", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 5, + ExitCode: 0, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 60, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should keep running a Pod continuously and produce log", func() { /* check the regular container restartCount > 0 */ + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, 0, 1, 2*time.Minute) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should restart a restartable init container before the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(restartableInit1, regular1)) + }) + ginkgo.It("should restart a restartable init container after the regular container started", func() { + framework.ExpectNoError(results.StartsBefore(regular1, restartableInit1)) + }) + ginkgo.It("should start a regular container", func() { + framework.ExpectNoError(results.Starts(regular1)) + }) + }) + + ginkgo.When("an Init container before restartable init container continuously fails", ginkgo.Ordered, func() { + + init1 := "init-1" + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "init-container-fails-before-restartable-init-starts", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: []v1.Container{ + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + }, + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should continuously run Pod keeping it Pending", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) { + if pod.Status.Phase != v1.PodPending { + return false, fmt.Errorf("pod should be in pending phase") + } + if len(pod.Status.InitContainerStatuses) < 1 { + return false, nil + } + containerStatus := pod.Status.InitContainerStatuses[0] + return containerStatus.RestartCount >= 3, nil + }) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should have Init container restartCount greater than 0", func() { + framework.ExpectNoError(results.HasRestarted(init1)) + }) + ginkgo.It("should not start restartable init container", func() { + framework.ExpectNoError(results.DoesntStart(restartableInit1)) + }) + }) + + ginkgo.When("an Init container after restartable init container fails", ginkgo.Ordered, func() { + + init1 := "init-1" + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-fails-before-init-container", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: init1, + Image: busyboxImage, + Command: ExecCommand(init1, execCommand{ + Delay: 1, + ExitCode: 1, + }), + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 600, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(podSpec) + var results containerOutputList + + ginkgo.It("should continuously run Pod keeping it Pending", func() { + client := e2epod.NewPodClient(f) + podSpec = client.Create(context.TODO(), podSpec) + + err := e2epod.WaitForPodCondition(context.TODO(), f.ClientSet, podSpec.Namespace, podSpec.Name, "pending and restarting 3 times", 5*time.Minute, func(pod *v1.Pod) (bool, error) { + if pod.Status.Phase != v1.PodPending { + return false, fmt.Errorf("pod should be in pending phase") + } + if len(pod.Status.InitContainerStatuses) < 1 { + return false, nil + } + containerStatus := pod.Status.InitContainerStatuses[0] + return containerStatus.RestartCount >= 3, nil + }) + framework.ExpectNoError(err) + + podSpec, err := client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results = parseOutput(podSpec) + }) + ginkgo.It("should have Init container restartCount greater than 0", func() { + framework.ExpectNoError(results.HasRestarted(init1)) + }) + // TODO: how will we be able to test it if restartable init container will never fail and there will be no termination log? Or will be? + ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { + framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) + }) + // TODO: check preStop hooks when they are enabled + }) + }) + + ginkgo.It("should launch restartable init cotnainers serially considering the startup probe", func() { + + restartableInit1 := "restartable-init-1" + restartableInit2 := "restartable-init-2" + regular1 := "regular-1" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-containers-start-serially", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + StartDelay: 10, + Delay: 600, + ExitCode: 0, + }), + StartupProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"test", "-f", "started"}, + }, + }, + }, + RestartPolicy: &containerRestartPolicyAlways, + }, + { + Name: restartableInit2, + Image: busyboxImage, + Command: ExecCommand(restartableInit2, execCommand{ + StartDelay: 10, + Delay: 600, + ExitCode: 0, + }), + StartupProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"test", "-f", "started"}, + }, + }, + }, + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(pod) + + client := e2epod.NewPodClient(f) + pod = client.Create(context.TODO(), pod) + + ginkgo.By("Waiting for the pod to finish") + err := e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, pod.Name, pod.Namespace, 5*time.Minute) + framework.ExpectNoError(err) + + pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + results := parseOutput(pod) + + ginkgo.By("Analyzing results") + framework.ExpectNoError(results.StartsBefore(restartableInit1, restartableInit2)) + framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1)) + }) + + ginkgo.It("should not launch next container if the restartable init cotnainer failed to complete startup probe", func() { + + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-failed-startup", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + StartDelay: 30, + Delay: 600, + ExitCode: 0, + }), + StartupProbe: &v1.Probe{ + PeriodSeconds: 1, + FailureThreshold: 1, + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"test", "-f", "started"}, + }, + }, + }, + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(pod) + + client := e2epod.NewPodClient(f) + pod = client.Create(context.TODO(), pod) + + ginkgo.By("Waiting for the restartable init cotnainer to restart") + err := WaitForPodInitContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 2, 2*time.Minute) + framework.ExpectNoError(err) + + pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + if pod.Status.Phase != v1.PodPending { + framework.Failf("pod %q is not pending, it's %q", pod.Name, pod.Status.Phase) + } + + results := parseOutput(pod) + + ginkgo.By("Analyzing results") + framework.ExpectNoError(results.DoesntStart(regular1)) + }) +})