Merge pull request #101960 from knight42/fix/deflake-metrics-proxy

Deflake tests that need to grab metrics from controller-manager or scheduler

test/e2e/framework/metrics/metrics_proxy.go

@@ -34,39 +34,54 @@ import (
 )
 
 type componentInfo struct {
+	Name string
 	Port int
 	IP   string
 }
 
 // SetupMetricsProxy creates a nginx Pod to expose metrics from the secure port of kube-scheduler and kube-controller-manager in tests.
 func SetupMetricsProxy(c clientset.Interface) error {
+	var infos []componentInfo
+	// The component pods might take some time to show up.
+	err := wait.PollImmediate(time.Second*5, time.Minute*5, func() (bool, error) {
 		podList, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{})
 		if err != nil {
-		return err
+			return false, fmt.Errorf("list pods in ns %s: %w", metav1.NamespaceSystem, err)
 		}
-	var infos []componentInfo
+		var foundComponents []componentInfo
 		for _, pod := range podList.Items {
+			if len(pod.Status.PodIP) == 0 {
+				continue
+			}
 			switch {
 			case strings.HasPrefix(pod.Name, "kube-scheduler-"):
-			infos = append(infos, componentInfo{
+				foundComponents = append(foundComponents, componentInfo{
+					Name: pod.Name,
 					Port: kubeSchedulerPort,
 					IP:   pod.Status.PodIP,
 				})
 			case strings.HasPrefix(pod.Name, "kube-controller-manager-"):
-			infos = append(infos, componentInfo{
+				foundComponents = append(foundComponents, componentInfo{
+					Name: pod.Name,
 					Port: kubeControllerManagerPort,
 					IP:   pod.Status.PodIP,
 				})
 			}
-		if len(infos) == 2 {
-			break
 		}
+		if len(foundComponents) != 2 {
+			klog.Infof("Only %d components found. Will retry.", len(foundComponents))
+			klog.Infof("Found components: %v", foundComponents)
+			return false, nil
 		}
-	if len(infos) == 0 {
+		infos = foundComponents
-		klog.Warningf("Can't find any pods in namespace %s to grab metrics from", metav1.NamespaceSystem)
+		return true, nil
-		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("missing component pods: %w", err)
 	}
 
+	klog.Infof("Found components: %v", infos)
+
 	const name = metricsProxyPod
 	_, err = c.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(context.TODO(), &v1.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{Name: name},