github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-29 06:27:05 +00:00
Code Issues Projects Releases Wiki Activity

Capture different parts of pod-startup latency as metrics

Browse Source
This commit is contained in:
Shyam Jeedigunta 2018-03-21 15:24:56 +01:00
parent 622ad35677
commit b0dd166fa3
4 changed files with 47 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
test/e2e/framework/metrics_util.go
View File

@ -188,7 +188,11 @@ type LatencyMetric struct {
} }
type PodStartupLatency struct { type PodStartupLatency struct {
Latency LatencyMetric `json:"latency"` CreateToScheduleLatency LatencyMetric `json:"createToScheduleLatency"`
ScheduleToRunLatency LatencyMetric `json:"scheduleToRunLatency"`
RunToWatchLatency LatencyMetric `json:"runToWatchLatency"`
ScheduleToWatchLatency LatencyMetric `json:"scheduleToWatchLatency"`
E2ELatency LatencyMetric `json:"e2eLatency"`
} }
func (l *PodStartupLatency) SummaryKind() string { func (l *PodStartupLatency) SummaryKind() string {
@ -398,17 +402,17 @@ func HighLatencyRequests(c clientset.Interface, nodeCount int) (int, *APIRespons
return badMetrics, metrics, nil return badMetrics, metrics, nil
} }
// Verifies whether 50, 90 and 99th percentiles of PodStartupLatency are // Verifies whether 50, 90 and 99th percentiles of e2e PodStartupLatency are
// within the threshold. // within the threshold.
func VerifyPodStartupLatency(latency *PodStartupLatency) error { func VerifyPodStartupLatency(latency *PodStartupLatency) error {
if latency.Latency.Perc50 > podStartupThreshold { if latency.E2ELatency.Perc50 > podStartupThreshold {
return fmt.Errorf("too high pod startup latency 50th percentile: %v", latency.Latency.Perc50) return fmt.Errorf("too high pod startup latency 50th percentile: %v", latency.E2ELatency.Perc50)
} }
if latency.Latency.Perc90 > podStartupThreshold { if latency.E2ELatency.Perc90 > podStartupThreshold {
return fmt.Errorf("too high pod startup latency 90th percentile: %v", latency.Latency.Perc90) return fmt.Errorf("too high pod startup latency 90th percentile: %v", latency.E2ELatency.Perc90)
} }
if latency.Latency.Perc99 > podStartupThreshold { if latency.E2ELatency.Perc99 > podStartupThreshold {
return fmt.Errorf("too high pod startup latency 99th percentile: %v", latency.Latency.Perc99) return fmt.Errorf("too high pod startup latency 99th percentile: %v", latency.E2ELatency.Perc99)
} }
return nil return nil
} }

27
test/e2e/framework/perf_util.go
View File

@ -53,22 +53,29 @@ func ApiCallToPerfData(apicalls *APIResponsiveness) *perftype.PerfData {
return perfData return perfData
} }
// PodStartupLatencyToPerfData transforms PodStartupLatency to PerfData. func latencyToPerfData(l LatencyMetric, name string) perftype.DataItem {
func PodStartupLatencyToPerfData(latency *PodStartupLatency) *perftype.PerfData { return perftype.DataItem{
perfData := &perftype.PerfData{Version: currentApiCallMetricsVersion}
item := perftype.DataItem{
Data: map[string]float64{ Data: map[string]float64{
"Perc50": float64(latency.Latency.Perc50) / 1000000, // us -> ms "Perc50": float64(l.Perc50) / 1000000, // us -> ms
"Perc90": float64(latency.Latency.Perc90) / 1000000, "Perc90": float64(l.Perc90) / 1000000,
"Perc99": float64(latency.Latency.Perc99) / 1000000, "Perc99": float64(l.Perc99) / 1000000,
"Perc100": float64(latency.Latency.Perc100) / 1000000, "Perc100": float64(l.Perc100) / 1000000,
}, },
Unit: "ms", Unit: "ms",
Labels: map[string]string{ Labels: map[string]string{
"Metric": "pod_startup", "Metric": name,
}, },
} }
perfData.DataItems = append(perfData.DataItems, item) }
// PodStartupLatencyToPerfData transforms PodStartupLatency to PerfData.
func PodStartupLatencyToPerfData(latency *PodStartupLatency) *perftype.PerfData {
perfData := &perftype.PerfData{Version: currentApiCallMetricsVersion}
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.CreateToScheduleLatency, "create_to_schedule"))
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.ScheduleToRunLatency, "schedule_to_run"))
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.RunToWatchLatency, "run_to_watch"))
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.ScheduleToWatchLatency, "schedule_to_watch"))
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.E2ELatency, "pod_startup"))
return perfData return perfData
} }

22
test/e2e/scalability/density.go
View File

@ -829,15 +829,23 @@ var _ = SIGDescribe("Density", func() {
sort.Sort(framework.LatencySlice(schedToWatchLag)) sort.Sort(framework.LatencySlice(schedToWatchLag))
sort.Sort(framework.LatencySlice(e2eLag)) sort.Sort(framework.LatencySlice(e2eLag))
framework.PrintLatencies(scheduleLag, "worst schedule latencies") framework.PrintLatencies(scheduleLag, "worst create-to-schedule latencies")
framework.PrintLatencies(startupLag, "worst run-after-schedule latencies") framework.PrintLatencies(startupLag, "worst schedule-to-run latencies")
framework.PrintLatencies(watchLag, "worst watch latencies") framework.PrintLatencies(watchLag, "worst run-to-watch latencies")
framework.PrintLatencies(schedToWatchLag, "worst scheduled-to-end total latencies") framework.PrintLatencies(schedToWatchLag, "worst schedule-to-watch latencies")
framework.PrintLatencies(e2eLag, "worst e2e total latencies") framework.PrintLatencies(e2eLag, "worst e2e latencies")
// Capture latency metrics related to pod-startup.
podStartupLatency := &framework.PodStartupLatency{
CreateToScheduleLatency: framework.ExtractLatencyMetrics(scheduleLag),
ScheduleToRunLatency: framework.ExtractLatencyMetrics(startupLag),
RunToWatchLatency: framework.ExtractLatencyMetrics(watchLag),
ScheduleToWatchLatency: framework.ExtractLatencyMetrics(schedToWatchLag),
E2ELatency: framework.ExtractLatencyMetrics(e2eLag),
}
f.TestSummaries = append(f.TestSummaries, podStartupLatency)
// Test whether e2e pod startup time is acceptable. // Test whether e2e pod startup time is acceptable.
podStartupLatency := &framework.PodStartupLatency{Latency: framework.ExtractLatencyMetrics(e2eLag)}
f.TestSummaries = append(f.TestSummaries, podStartupLatency)
framework.ExpectNoError(framework.VerifyPodStartupLatency(podStartupLatency)) framework.ExpectNoError(framework.VerifyPodStartupLatency(podStartupLatency))
framework.LogSuspiciousLatency(startupLag, e2eLag, nodeCount, c) framework.LogSuspiciousLatency(startupLag, e2eLag, nodeCount, c)

6
test/e2e_node/density_test.go
View File

@ -556,14 +556,14 @@ func logAndVerifyLatency(batchLag time.Duration, e2eLags []framework.PodLatencyD
latencyMetrics, _ := getPodStartLatency(kubeletAddr) latencyMetrics, _ := getPodStartLatency(kubeletAddr)
framework.Logf("Kubelet Prometheus metrics (not reset):\n%s", framework.PrettyPrintJSON(latencyMetrics)) framework.Logf("Kubelet Prometheus metrics (not reset):\n%s", framework.PrettyPrintJSON(latencyMetrics))
podCreateLatency := framework.PodStartupLatency{Latency: framework.ExtractLatencyMetrics(e2eLags)} podStartupLatency := framework.ExtractLatencyMetrics(e2eLags)
// log latency perf data // log latency perf data
logPerfData(getLatencyPerfData(podCreateLatency.Latency, testInfo), "latency") logPerfData(getLatencyPerfData(podStartupLatency, testInfo), "latency")
if isVerify { if isVerify {
// check whether e2e pod startup time is acceptable. // check whether e2e pod startup time is acceptable.
framework.ExpectNoError(verifyPodStartupLatency(podStartupLimits, podCreateLatency.Latency)) framework.ExpectNoError(verifyPodStartupLatency(podStartupLimits, podStartupLatency))
// check bactch pod creation latency // check bactch pod creation latency
if podBatchStartupLimit > 0 { if podBatchStartupLimit > 0 {