Merge pull request #60906 from MrHohn/e2e-restart-apiserver-refine

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

[e2e service] Refine apiserver restart logic

**What this PR does / why we need it**:
Ref https://github.com/kubernetes/kubernetes/issues/60761#issuecomment-371308569, wait for apiserver's restart count increases before proceeding the test.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes (hopefully) #60761

**Special notes for your reviewer**:
/assign @rramkumar1 @bowei 

**Release note**:

```release-note
NONE
```

test/e2e/framework/util.go

@@ -3962,6 +3962,42 @@ func WaitForApiserverUp(c clientset.Interface) error {
 	return fmt.Errorf("waiting for apiserver timed out")
 }
 
+// WaitForApiserverRestarted waits until apiserver's restart count increased.
+func WaitForApiserverRestarted(c clientset.Interface, initialRestartCount int32) error {
+	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) {
+		restartCount, err := GetApiserverRestartCount(c)
+		if err != nil {
+			Logf("Failed to get apiserver's restart count: %v", err)
+			continue
+		}
+		if restartCount > initialRestartCount {
+			Logf("Apiserver has restarted.")
+			return nil
+		}
+		Logf("Waiting for apiserver restart count to increase")
+	}
+	return fmt.Errorf("timed out waiting for apiserver to be restarted")
+}
+
+func GetApiserverRestartCount(c clientset.Interface) (int32, error) {
+	label := labels.SelectorFromSet(labels.Set(map[string]string{"component": "kube-apiserver"}))
+	listOpts := metav1.ListOptions{LabelSelector: label.String()}
+	pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts)
+	if err != nil {
+		return -1, err
+	}
+	if len(pods.Items) != 1 {
+		return -1, fmt.Errorf("unexpected number of apiserver pod: %d", len(pods.Items))
+	}
+	for _, s := range pods.Items[0].Status.ContainerStatuses {
+		if s.Name != "kube-apiserver" {
+			continue
+		}
+		return s.RestartCount, nil
+	}
+	return -1, fmt.Errorf("failed to find kube-apiserver container in pod")
+}
+
 func RestartControllerManager() error {
 	// TODO: Make it work for all providers and distros.
 	if !ProviderIs("gce", "aws") {

test/e2e/network/service.go

@@ -427,10 +427,16 @@ var _ = SIGDescribe("Services", func() {
 		framework.ExpectNoError(framework.VerifyServeHostnameServiceUp(cs, ns, host, podNames1, svc1IP, servicePort))
 
 		// Restart apiserver
+		initialRestartCount, err := framework.GetApiserverRestartCount(cs)
+		Expect(err).NotTo(HaveOccurred(), "failed to get apiserver's restart count")
 		By("Restarting apiserver")
 		if err := framework.RestartApiserver(cs.Discovery()); err != nil {
 			framework.Failf("error restarting apiserver: %v", err)
 		}
+		By("Waiting for apiserver to be restarted")
+		if err := framework.WaitForApiserverRestarted(cs, initialRestartCount); err != nil {
+			framework.Failf("error while waiting for apiserver to be restarted: %v", err)
+		}
 		By("Waiting for apiserver to come up by polling /healthz")
 		if err := framework.WaitForApiserverUp(cs); err != nil {
 			framework.Failf("error while waiting for apiserver up: %v", err)