From b203fb0565607f37f907cc89e2d257e93ae4e9b8 Mon Sep 17 00:00:00 2001
From: Matthias Bertschy <matthias.bertschy@gmail.com>
Date: Sat, 6 Mar 2021 11:58:03 +0100
Subject: [PATCH] Deflake e2e test for startupProbe

---
 test/e2e/common/node/container_probe.go | 24 ++++++++++++------------
 test/e2e/framework/pod/resource.go      | 14 ++++++++++++++
 test/e2e/framework/pod/wait.go          |  5 +++++
 3 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/test/e2e/common/node/container_probe.go b/test/e2e/common/node/container_probe.go
index 7f5d81ca2ff..20aa75b8e87 100644
--- a/test/e2e/common/node/container_probe.go
+++ b/test/e2e/common/node/container_probe.go
@@ -374,10 +374,9 @@ var _ = SIGDescribe("Probing container", func() {
 		Description: A Pod is created with startup and readiness probes. The Container is started by creating /tmp/startup after 45 seconds, delaying the ready state by this amount of time. This is similar to the "Pod readiness probe, with initial delay" test.
 	*/
 	ginkgo.It("should be ready immediately after startupProbe succeeds", func() {
-		sleepBeforeStarted := time.Duration(45)
-		cmd := []string{"/bin/sh", "-c", fmt.Sprintf("sleep %d; echo ok >/tmp/startup; sleep 600", sleepBeforeStarted)}
+		cmd := []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 10; echo ok >/tmp/startup; sleep 600"}
 		readinessProbe := &v1.Probe{
-			Handler:             execHandler([]string{"/bin/true"}),
+			Handler:             execHandler([]string{"/bin/cat", "/tmp/health"}),
 			InitialDelaySeconds: 0,
 			PeriodSeconds:       60,
 		}
@@ -391,7 +390,15 @@ var _ = SIGDescribe("Probing container", func() {
 		p, err := podClient.Get(context.TODO(), p.Name, metav1.GetOptions{})
 		framework.ExpectNoError(err)
 
-		e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, p.Name, f.Namespace.Name, framework.PodStartTimeout)
+		err = e2epod.WaitForPodContainerStarted(f.ClientSet, f.Namespace.Name, p.Name, 0, framework.PodStartTimeout)
+		framework.ExpectNoError(err)
+		startedTime := time.Now()
+
+		// We assume the pod became ready when the container became ready. This
+		// is true for a single container pod.
+		err = e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, p.Name, f.Namespace.Name, framework.PodStartTimeout)
+		framework.ExpectNoError(err)
+		readyTime := time.Now()
 
 		p, err = podClient.Get(context.TODO(), p.Name, metav1.GetOptions{})
 		framework.ExpectNoError(err)
@@ -400,14 +407,7 @@ var _ = SIGDescribe("Probing container", func() {
 		framework.ExpectNoError(err)
 		framework.ExpectEqual(isReady, true, "pod should be ready")
 
-		// We assume the pod became ready when the container became ready. This
-		// is true for a single container pod.
-		readyTime, err := GetTransitionTimeForReadyCondition(p)
-		framework.ExpectNoError(err)
-		startedTime, err := GetContainerStartedTime(p, "busybox")
-		framework.ExpectNoError(err)
-
-		readyIn := readyTime.Sub(startedTime) - sleepBeforeStarted*time.Second
+		readyIn := readyTime.Sub(startedTime)
 		framework.Logf("Container started at %v, pod became ready at %v, %v after startupProbe succeeded", startedTime, readyTime, readyIn)
 		if readyIn < 0 {
 			framework.Failf("Pod became ready before startupProbe succeeded")
diff --git a/test/e2e/framework/pod/resource.go b/test/e2e/framework/pod/resource.go
index 668cf263ad7..23f4c565b9f 100644
--- a/test/e2e/framework/pod/resource.go
+++ b/test/e2e/framework/pod/resource.go
@@ -321,6 +321,20 @@ func podContainerFailed(c clientset.Interface, namespace, podName string, contai
 	}
 }
 
+func podContainerStarted(c clientset.Interface, namespace, podName string, containerIndex int) wait.ConditionFunc {
+	return func() (bool, error) {
+		pod, err := c.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if containerIndex > len(pod.Status.ContainerStatuses)-1 {
+			return false, nil
+		}
+		containerStatus := pod.Status.ContainerStatuses[containerIndex]
+		return *containerStatus.Started, nil
+	}
+}
+
 // LogPodStates logs basic info of provided pods for debugging.
 func LogPodStates(pods []v1.Pod) {
 	// Find maximum widths for pod, node, and phase strings for column printing.
diff --git a/test/e2e/framework/pod/wait.go b/test/e2e/framework/pod/wait.go
index cb107a920b8..6de8e87363e 100644
--- a/test/e2e/framework/pod/wait.go
+++ b/test/e2e/framework/pod/wait.go
@@ -542,3 +542,8 @@ func WaitForNRestartablePods(ps *testutils.PodStore, expect int, timeout time.Du
 func WaitForPodContainerToFail(c clientset.Interface, namespace, podName string, containerIndex int, reason string, timeout time.Duration) error {
 	return wait.PollImmediate(poll, timeout, podContainerFailed(c, namespace, podName, containerIndex, reason))
 }
+
+// WaitForPodContainerStarted waits for the given Pod container to start, after a successful run of the startupProbe.
+func WaitForPodContainerStarted(c clientset.Interface, namespace, podName string, containerIndex int, timeout time.Duration) error {
+	return wait.PollImmediate(poll, timeout, podContainerStarted(c, namespace, podName, containerIndex))
+}