node-e2e: Add container lifecycle e2e tests for preStop hook

This ensures that the container's pre-stop hook is invoked if the
startup or liveness probe fails.

test/e2e_node/container_lifecycle_pod_construction.go

@@ -38,6 +38,9 @@ type execCommand struct {
 	StartDelay int
 	// Delay is how long the container should delay before exiting
 	Delay int
+	// TerminationSeconds is the time it takes for the container before
+	// terminating if it catches SIGTERM.
+	TerminationSeconds int
 }
 
 // ExecCommand returns the command to execute in the container that implements execCommand and logs activities to a container
@@ -57,21 +60,30 @@ func ExecCommand(name string, c execCommand) []string {
 	fmt.Fprintf(&cmd, "cat %s >> /dev/termination-log; ", containerLog)
 
 	fmt.Fprintf(&cmd, "echo %s '%s Starting %d' | tee -a %s >> /dev/termination-log; ", timeCmd, name, c.StartDelay, containerLog)
+	fmt.Fprintf(&cmd, "_term() { sleep %d; echo %s '%s Exiting' | tee -a %s >> /dev/termination-log; exit %d; }; ", c.TerminationSeconds, timeCmd, name, containerLog, c.ExitCode)
+	fmt.Fprintf(&cmd, "trap _term TERM; ")
 	if c.StartDelay != 0 {
-		fmt.Fprintf(&cmd, "sleep %d; ", c.StartDelay)
+		fmt.Fprint(&cmd, sleepCommand(c.StartDelay))
 	}
 	// You can check started file to see if the container has started
 	fmt.Fprintf(&cmd, "touch started; ")
 	fmt.Fprintf(&cmd, "echo %s '%s Started' | tee -a %s >> /dev/termination-log; ", timeCmd, name, containerLog)
 	fmt.Fprintf(&cmd, "echo %s '%s Delaying %d' | tee -a %s >> /dev/termination-log; ", timeCmd, name, c.Delay, containerLog)
 	if c.Delay != 0 {
-		fmt.Fprintf(&cmd, "sleep %d; ", c.Delay)
+		fmt.Fprint(&cmd, sleepCommand(c.Delay))
 	}
 	fmt.Fprintf(&cmd, "echo %s '%s Exiting'  | tee -a %s >> /dev/termination-log; ", timeCmd, name, containerLog)
 	fmt.Fprintf(&cmd, "exit %d", c.ExitCode)
 	return []string{"sh", "-c", cmd.String()}
 }
 
+// sleepCommand returns a command that sleeps for the given number of seconds
+// in background and waits for it to finish so that the parent process can
+// handle signals.
+func sleepCommand(seconds int) string {
+	return fmt.Sprintf("exec sleep %d & wait $!; ", seconds)
+}
+
 type containerOutput struct {
 	// time the message was seen to the nearest second
 	timestamp time.Time

test/e2e_node/container_lifecycle_test.go

@@ -33,6 +33,7 @@ import (
 
 const (
 	PostStartPrefix = "PostStart"
+	PreStopPrefix   = "PreStop"
 )
 
 var containerRestartPolicyAlways = v1.ContainerRestartPolicyAlways
@@ -596,6 +597,138 @@ var _ = SIGDescribe("[NodeConformance] Containers Lifecycle ", func() {
 			framework.ExpectNoError(err)
 		})
 	})
+
+	ginkgo.It("should call the container's preStop hook and terminate it if its startup probe fails", func() {
+		regular1 := "regular-1"
+
+		podSpec := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-pod",
+			},
+			Spec: v1.PodSpec{
+				RestartPolicy: v1.RestartPolicyNever,
+				Containers: []v1.Container{
+					{
+						Name:  regular1,
+						Image: busyboxImage,
+						Command: ExecCommand(regular1, execCommand{
+							Delay:              100,
+							TerminationSeconds: 15,
+							ExitCode:           0,
+						}),
+						StartupProbe: &v1.Probe{
+							ProbeHandler: v1.ProbeHandler{
+								Exec: &v1.ExecAction{
+									Command: []string{
+										"sh",
+										"-c",
+										"exit 1",
+									},
+								},
+							},
+							InitialDelaySeconds: 10,
+							FailureThreshold:    1,
+						},
+						Lifecycle: &v1.Lifecycle{
+							PreStop: &v1.LifecycleHandler{
+								Exec: &v1.ExecAction{
+									Command: ExecCommand(prefixedName(PreStopPrefix, regular1), execCommand{
+										Delay:    1,
+										ExitCode: 0,
+									}),
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		preparePod(podSpec)
+
+		client := e2epod.NewPodClient(f)
+		podSpec = client.Create(context.TODO(), podSpec)
+
+		ginkgo.By("Waiting for the pod to complete")
+		err := e2epod.WaitForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Parsing results")
+		podSpec, err = client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
+		framework.ExpectNoError(err)
+		results := parseOutput(podSpec)
+
+		ginkgo.By("Analyzing results")
+		framework.ExpectNoError(results.RunTogether(regular1, prefixedName(PreStopPrefix, regular1)))
+		framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, regular1)))
+		framework.ExpectNoError(results.Exits(regular1))
+	})
+
+	ginkgo.It("should call the container's preStop hook and terminate it if its liveness probe fails", func() {
+		regular1 := "regular-1"
+
+		podSpec := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-pod",
+			},
+			Spec: v1.PodSpec{
+				RestartPolicy: v1.RestartPolicyNever,
+				Containers: []v1.Container{
+					{
+						Name:  regular1,
+						Image: busyboxImage,
+						Command: ExecCommand(regular1, execCommand{
+							Delay:              100,
+							TerminationSeconds: 15,
+							ExitCode:           0,
+						}),
+						LivenessProbe: &v1.Probe{
+							ProbeHandler: v1.ProbeHandler{
+								Exec: &v1.ExecAction{
+									Command: []string{
+										"sh",
+										"-c",
+										"exit 1",
+									},
+								},
+							},
+							InitialDelaySeconds: 10,
+							FailureThreshold:    1,
+						},
+						Lifecycle: &v1.Lifecycle{
+							PreStop: &v1.LifecycleHandler{
+								Exec: &v1.ExecAction{
+									Command: ExecCommand(prefixedName(PreStopPrefix, regular1), execCommand{
+										Delay:    1,
+										ExitCode: 0,
+									}),
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		preparePod(podSpec)
+
+		client := e2epod.NewPodClient(f)
+		podSpec = client.Create(context.TODO(), podSpec)
+
+		ginkgo.By("Waiting for the pod to complete")
+		err := e2epod.WaitForPodNoLongerRunningInNamespace(context.TODO(), f.ClientSet, podSpec.Name, podSpec.Namespace)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Parsing results")
+		podSpec, err = client.Get(context.TODO(), podSpec.Name, metav1.GetOptions{})
+		framework.ExpectNoError(err)
+		results := parseOutput(podSpec)
+
+		ginkgo.By("Analyzing results")
+		framework.ExpectNoError(results.RunTogether(regular1, prefixedName(PreStopPrefix, regular1)))
+		framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, regular1)))
+		framework.ExpectNoError(results.Exits(regular1))
+	})
 })
 
 var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle ", func() {