From e1f4d5b11091f0154390e74e520c8d517b6e1d07 Mon Sep 17 00:00:00 2001
From: Kenichi Omichi <ken-oomichi@wx.jp.nec.com>
Date: Tue, 30 Jun 2020 00:06:37 +0000
Subject: [PATCH] Add nodeHasControlPlanePods()

DeprecatedMightBeMasterNode() has been marked as deprecated and we need to
find alternative way for callers of the function.
In NewResourceUsageGatherer(), the function was called for distinguishing
the specified pods are running on master nodes, and the gatherer gathers
those pods' resource usage.
This adds nodeHasControlPlanePods() to gistinguish the specified pods
are running on nodes which are operating control plane pods (kube-scheduler
and kube-controller-manager) and replace callers of DeprecatedMightBeMasterNode()
with this new function as better way.
---
 test/e2e/framework/BUILD                      |  1 -
 test/e2e/framework/resource_usage_gatherer.go | 52 ++++++++++++++++---
 2 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD
index 2a79fd0ac56..3f7567b62dc 100644
--- a/test/e2e/framework/BUILD
+++ b/test/e2e/framework/BUILD
@@ -65,7 +65,6 @@ go_library(
         "//test/e2e/framework/node:go_default_library",
         "//test/e2e/framework/pod:go_default_library",
         "//test/e2e/framework/ssh:go_default_library",
-        "//test/e2e/system:go_default_library",
         "//test/utils:go_default_library",
         "//test/utils/image:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
diff --git a/test/e2e/framework/resource_usage_gatherer.go b/test/e2e/framework/resource_usage_gatherer.go
index 757b7a06e58..4673b60d133 100644
--- a/test/e2e/framework/resource_usage_gatherer.go
+++ b/test/e2e/framework/resource_usage_gatherer.go
@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -32,10 +33,10 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeletstatsv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
-	"k8s.io/kubernetes/test/e2e/system"
 
 	// TODO: Remove the following imports (ref: https://github.com/kubernetes/kubernetes/issues/81245)
 	e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
@@ -370,6 +371,29 @@ const (
 	MasterAndDNSNodes NodesSet = 2
 )
 
+// nodeHasControlPlanePods returns true if specified node has control plane pods
+// (kube-scheduler and/or kube-controller-manager).
+func nodeHasControlPlanePods(c clientset.Interface, nodeName string) (bool, error) {
+	regKubeScheduler := regexp.MustCompile("kube-scheduler-.*")
+	regKubeControllerManager := regexp.MustCompile("kube-controller-manager-.*")
+
+	podList, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String(),
+	})
+	if err != nil {
+		return false, err
+	}
+	if len(podList.Items) < 1 {
+		Logf("Can't find any pods in namespace %s to grab metrics from", metav1.NamespaceSystem)
+	}
+	for _, pod := range podList.Items {
+		if regKubeScheduler.MatchString(pod.Name) || regKubeControllerManager.MatchString(pod.Name) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 // NewResourceUsageGatherer returns a new ContainerResourceGatherer.
 func NewResourceUsageGatherer(c clientset.Interface, options ResourceGathererOptions, pods *v1.PodList) (*ContainerResourceGatherer, error) {
 	g := ContainerResourceGatherer{
@@ -404,11 +428,23 @@ func NewResourceUsageGatherer(c clientset.Interface, options ResourceGathererOpt
 	}
 	dnsNodes := make(map[string]bool)
 	for _, pod := range pods.Items {
-		if (options.Nodes == MasterNodes) && !system.DeprecatedMightBeMasterNode(pod.Spec.NodeName) {
-			continue
+		if options.Nodes == MasterNodes {
+			isControlPlane, err := nodeHasControlPlanePods(c, pod.Spec.NodeName)
+			if err != nil {
+				return nil, err
+			}
+			if !isControlPlane {
+				continue
+			}
 		}
-		if (options.Nodes == MasterAndDNSNodes) && !system.DeprecatedMightBeMasterNode(pod.Spec.NodeName) && pod.Labels["k8s-app"] != "kube-dns" {
-			continue
+		if options.Nodes == MasterAndDNSNodes {
+			isControlPlane, err := nodeHasControlPlanePods(c, pod.Spec.NodeName)
+			if err != nil {
+				return nil, err
+			}
+			if !isControlPlane && pod.Labels["k8s-app"] != "kube-dns" {
+				continue
+			}
 		}
 		for _, container := range pod.Status.InitContainerStatuses {
 			g.containerIDs = append(g.containerIDs, container.Name)
@@ -427,7 +463,11 @@ func NewResourceUsageGatherer(c clientset.Interface, options ResourceGathererOpt
 	}
 
 	for _, node := range nodeList.Items {
-		if options.Nodes == AllNodes || system.DeprecatedMightBeMasterNode(node.Name) || dnsNodes[node.Name] {
+		isControlPlane, err := nodeHasControlPlanePods(c, node.Name)
+		if err != nil {
+			return nil, err
+		}
+		if options.Nodes == AllNodes || isControlPlane || dnsNodes[node.Name] {
 			g.workerWg.Add(1)
 			g.workers = append(g.workers, resourceGatherWorker{
 				c:                           c,