Merge pull request #46053 from dashpole/test_eviction_metrics

Automatic merge from submit-queue (batch tested with PRs 46033, 46122, 46053, 46018, 45981)

Log age of stats used for evictions during eviction tests

I recently added prometheus metrics for the age of the metrics used for evictions #43031.  It would be nice to surface these during eviction tests, so I can better assess how old stats are, and whether or not the age of stats causes extra evictions.

This isnt super-high priority, and can be done after code-freeze, since it is a testing improvement.  Feel free to take a look whenever either of you has time.

/assign @mtaufen 
/assign @Random-Liu

test/e2e/framework/kubelet_stats.go

@@ -109,9 +109,13 @@ func GetKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMetrics {
 		kubeletmetrics.PodWorkerStartLatencyKey,
 		kubeletmetrics.PLEGRelistLatencyKey,
 	)
+	return GetKubeletMetrics(ms, latencyMethods)
+}
+
+func GetKubeletMetrics(ms metrics.KubeletMetrics, methods sets.String) KubeletLatencyMetrics {
 	var latencyMetrics KubeletLatencyMetrics
 	for method, samples := range ms {
-		if !latencyMethods.Has(method) {
+		if !methods.Has(method) {
 			continue
 		}
 		for _, sample := range samples {

test/e2e_node/BUILD

@@ -34,7 +34,9 @@ go_library(
         "//pkg/kubelet/apis/cri:go_default_library",
         "//pkg/kubelet/apis/cri/v1alpha1:go_default_library",
         "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
+        "//pkg/kubelet/metrics:go_default_library",
         "//pkg/kubelet/remote:go_default_library",
+        "//pkg/metrics:go_default_library",
         "//pkg/util/procfs:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",

test/e2e_node/inode_eviction_test.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/kubernetes/pkg/api/v1"
 	nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
@@ -175,6 +176,7 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
 			for _, p := range updatedPods {
 				framework.Logf("fetching pod %s; phase= %v", p.Name, p.Status.Phase)
 			}
+			logKubeletMetrics(kubeletmetrics.EvictionStatsAgeKey)
 			_, err = hasPressureCondition(f, testCondition)
 			if err != nil {
 				return err
@@ -261,6 +263,7 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
 				if priorityPodSpec.evictionPriority == 0 {
 					for _, p := range updatedPodList.Items {
 						if p.Name == priorityPodSpec.pod.Name && p.Status.Phase == v1.PodFailed {
+							logKubeletMetrics(kubeletmetrics.EvictionStatsAgeKey)
 							return fmt.Errorf("%s pod failed (delayed) and shouldn't have failed", p.Name)
 						}
 					}

test/e2e_node/util.go

@@ -30,12 +30,14 @@ import (
 
 	k8serr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	v1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
-	// utilconfig "k8s.io/kubernetes/pkg/util/config"
+	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
+	"k8s.io/kubernetes/pkg/metrics"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
@@ -292,3 +294,17 @@ func getLocalNode(f *framework.Framework) *v1.Node {
 	Expect(len(nodeList.Items)).To(Equal(1), "Unexpected number of node objects for node e2e. Expects only one node.")
 	return &nodeList.Items[0]
 }
+
+// logs prometheus metrics from the local kubelet.
+func logKubeletMetrics(metricKeys ...string) {
+	metricSet := sets.NewString()
+	for _, key := range metricKeys {
+		metricSet.Insert(kubeletmetrics.KubeletSubsystem + "_" + key)
+	}
+	metric, err := metrics.GrabKubeletMetricsWithoutProxy(framework.TestContext.NodeName + ":10255")
+	if err != nil {
+		framework.Logf("Error getting kubelet metrics: %v", err)
+	} else {
+		framework.Logf("Kubelet Metrics: %+v", framework.GetKubeletMetrics(metric, metricSet))
+	}
+}