github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 12:43:23 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #46053 from dashpole/test_eviction_metrics

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 46033, 46122, 46053, 46018, 45981)

Log age of stats used for evictions during eviction tests

I recently added prometheus metrics for the age of the metrics used for evictions #43031.  It would be nice to surface these during eviction tests, so I can better assess how old stats are, and whether or not the age of stats causes extra evictions.

This isnt super-high priority, and can be done after code-freeze, since it is a testing improvement.  Feel free to take a look whenever either of you has time.

/assign @mtaufen 
/assign @Random-Liu
This commit is contained in:
Kubernetes Submit Queue 2017-05-19 23:29:28 -07:00 committed by GitHub
commit 112ed869c7
4 changed files with 27 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
test/e2e/framework/kubelet_stats.go
View File

@ -109,9 +109,13 @@ func GetKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMetrics {
kubeletmetrics.PodWorkerStartLatencyKey,
kubeletmetrics.PLEGRelistLatencyKey,
)
return GetKubeletMetrics(ms, latencyMethods)
}
func GetKubeletMetrics(ms metrics.KubeletMetrics, methods sets.String) KubeletLatencyMetrics {
var latencyMetrics KubeletLatencyMetrics
for method, samples := range ms {
if !latencyMethods.Has(method) {
if !methods.Has(method) {
continue
}
for _, sample := range samples {

2
test/e2e_node/BUILD
View File

@ -34,7 +34,9 @@ go_library(
"//pkg/kubelet/apis/cri:go_default_library",
"//pkg/kubelet/apis/cri/v1alpha1:go_default_library",
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
"//pkg/kubelet/metrics:go_default_library",
"//pkg/kubelet/remote:go_default_library",
"//pkg/metrics:go_default_library",
"//pkg/util/procfs:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",

3
test/e2e_node/inode_eviction_test.go
View File

@ -24,6 +24,7 @@ import (
"k8s.io/kubernetes/pkg/api/v1"
nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
"k8s.io/kubernetes/pkg/apis/componentconfig"
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
@ -175,6 +176,7 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
for _, p := range updatedPods {
framework.Logf("fetching pod %s; phase= %v", p.Name, p.Status.Phase)
}
logKubeletMetrics(kubeletmetrics.EvictionStatsAgeKey)
_, err = hasPressureCondition(f, testCondition)
if err != nil {
return err
@ -261,6 +263,7 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
if priorityPodSpec.evictionPriority == 0 {
for _, p := range updatedPodList.Items {
if p.Name == priorityPodSpec.pod.Name && p.Status.Phase == v1.PodFailed {
logKubeletMetrics(kubeletmetrics.EvictionStatsAgeKey)
return fmt.Errorf("%s pod failed (delayed) and shouldn't have failed", p.Name)
}
}

18
test/e2e_node/util.go
View File

@ -30,12 +30,14 @@ import (
k8serr "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/apis/componentconfig"
v1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
// utilconfig "k8s.io/kubernetes/pkg/util/config"
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/metrics"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
@ -292,3 +294,17 @@ func getLocalNode(f *framework.Framework) *v1.Node {
Expect(len(nodeList.Items)).To(Equal(1), "Unexpected number of node objects for node e2e. Expects only one node.")
return &nodeList.Items[0]
}
// logs prometheus metrics from the local kubelet.
func logKubeletMetrics(metricKeys ...string) {
metricSet := sets.NewString()
for _, key := range metricKeys {
metricSet.Insert(kubeletmetrics.KubeletSubsystem + "_" + key)
}
metric, err := metrics.GrabKubeletMetricsWithoutProxy(framework.TestContext.NodeName + ":10255")
if err != nil {
framework.Logf("Error getting kubelet metrics: %v", err)
} else {
framework.Logf("Kubelet Metrics: %+v", framework.GetKubeletMetrics(metric, metricSet))
}
}