github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-06 10:43:56 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #52732 from shyamjvs/fix-metrics-perf-tests

Browse Source 
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>..

Increase api latency threshold for cluster-scoped list calls

Recent change from @smarterclayton (https://github.com/kubernetes/kubernetes/pull/52237) added scope to apiserver metrics. As a result, our current threshold for list calls is no longer sufficient for all-namespace calls which are now being measured separately from namespaced lists. For e.g (from our [last 5k run](https://k8s-gubernator.appspot.com/build/kubernetes-jenkins/logs/ci-kubernetes-e2e-gce-scale-performance/37)):

```
WARNING Top latency metric: {Resource:pods Subresource: Verb:LIST Scope:cluster Latency:{Perc50:4.498374s Perc90:7.548079s Perc99:8.169389s Perc100:0s} Count:1400}
```

cc @kubernetes/sig-scalability-misc @kubernetes/sig-api-machinery-misc @wojtek-t
This commit is contained in:
Kubernetes Submit Queue 2017-09-21 10:49:54 -07:00 committed by GitHub
commit 542486186f
1 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
test/e2e/framework/metrics_util.go
View File

@ -52,9 +52,11 @@ const (
apiCallLatencyThreshold time.Duration = 1 * time.Second apiCallLatencyThreshold time.Duration = 1 * time.Second
// We use a higher threshold for list apicalls if the cluster is big (i.e having > 500 nodes) // We use a higher threshold for list apicalls if the cluster is big (i.e having > 500 nodes)
// as list response sizes are bigger in general for big clusters. // as list response sizes are bigger in general for big clusters. We also use a higher threshold
apiListCallLatencyThreshold time.Duration = 5 * time.Second // for list calls at cluster scope (this includes non-namespaced and all-namespaced calls).
bigClusterNodeCountThreshold = 500 apiListCallLatencyThreshold time.Duration = 5 * time.Second
apiClusterScopeListCallThreshold time.Duration = 10 * time.Second
bigClusterNodeCountThreshold = 500
// Cluster Autoscaler metrics names // Cluster Autoscaler metrics names
caFunctionMetric = "cluster_autoscaler_function_duration_seconds_bucket" caFunctionMetric = "cluster_autoscaler_function_duration_seconds_bucket"
@ -371,13 +373,19 @@ func HighLatencyRequests(c clientset.Interface, nodeCount int) (int, *APIRespons
for i := range metrics.APICalls { for i := range metrics.APICalls {
latency := metrics.APICalls[i].Latency.Perc99 latency := metrics.APICalls[i].Latency.Perc99
isListCall := (metrics.APICalls[i].Verb == "LIST") isListCall := (metrics.APICalls[i].Verb == "LIST")
isClusterScopedCall := (metrics.APICalls[i].Scope == "cluster")
isBad := false isBad := false
if latency > apiCallLatencyThreshold { latencyThreshold := apiCallLatencyThreshold
if !isListCall || !isBigCluster || (latency > apiListCallLatencyThreshold) { if isListCall && isBigCluster {
isBad = true latencyThreshold = apiListCallLatencyThreshold
badMetrics++ if isClusterScopedCall {
latencyThreshold = apiClusterScopeListCallThreshold
} }
} }
if latency > latencyThreshold {
isBad = true
badMetrics++
}
if top > 0 || isBad { if top > 0 || isBad {
top-- top--
prefix := "" prefix := ""