mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-28 14:07:14 +00:00
Capture different parts of pod-startup latency as metrics
This commit is contained in:
parent
622ad35677
commit
b0dd166fa3
@ -188,7 +188,11 @@ type LatencyMetric struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PodStartupLatency struct {
|
type PodStartupLatency struct {
|
||||||
Latency LatencyMetric `json:"latency"`
|
CreateToScheduleLatency LatencyMetric `json:"createToScheduleLatency"`
|
||||||
|
ScheduleToRunLatency LatencyMetric `json:"scheduleToRunLatency"`
|
||||||
|
RunToWatchLatency LatencyMetric `json:"runToWatchLatency"`
|
||||||
|
ScheduleToWatchLatency LatencyMetric `json:"scheduleToWatchLatency"`
|
||||||
|
E2ELatency LatencyMetric `json:"e2eLatency"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *PodStartupLatency) SummaryKind() string {
|
func (l *PodStartupLatency) SummaryKind() string {
|
||||||
@ -398,17 +402,17 @@ func HighLatencyRequests(c clientset.Interface, nodeCount int) (int, *APIRespons
|
|||||||
return badMetrics, metrics, nil
|
return badMetrics, metrics, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verifies whether 50, 90 and 99th percentiles of PodStartupLatency are
|
// Verifies whether 50, 90 and 99th percentiles of e2e PodStartupLatency are
|
||||||
// within the threshold.
|
// within the threshold.
|
||||||
func VerifyPodStartupLatency(latency *PodStartupLatency) error {
|
func VerifyPodStartupLatency(latency *PodStartupLatency) error {
|
||||||
if latency.Latency.Perc50 > podStartupThreshold {
|
if latency.E2ELatency.Perc50 > podStartupThreshold {
|
||||||
return fmt.Errorf("too high pod startup latency 50th percentile: %v", latency.Latency.Perc50)
|
return fmt.Errorf("too high pod startup latency 50th percentile: %v", latency.E2ELatency.Perc50)
|
||||||
}
|
}
|
||||||
if latency.Latency.Perc90 > podStartupThreshold {
|
if latency.E2ELatency.Perc90 > podStartupThreshold {
|
||||||
return fmt.Errorf("too high pod startup latency 90th percentile: %v", latency.Latency.Perc90)
|
return fmt.Errorf("too high pod startup latency 90th percentile: %v", latency.E2ELatency.Perc90)
|
||||||
}
|
}
|
||||||
if latency.Latency.Perc99 > podStartupThreshold {
|
if latency.E2ELatency.Perc99 > podStartupThreshold {
|
||||||
return fmt.Errorf("too high pod startup latency 99th percentile: %v", latency.Latency.Perc99)
|
return fmt.Errorf("too high pod startup latency 99th percentile: %v", latency.E2ELatency.Perc99)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -53,22 +53,29 @@ func ApiCallToPerfData(apicalls *APIResponsiveness) *perftype.PerfData {
|
|||||||
return perfData
|
return perfData
|
||||||
}
|
}
|
||||||
|
|
||||||
// PodStartupLatencyToPerfData transforms PodStartupLatency to PerfData.
|
func latencyToPerfData(l LatencyMetric, name string) perftype.DataItem {
|
||||||
func PodStartupLatencyToPerfData(latency *PodStartupLatency) *perftype.PerfData {
|
return perftype.DataItem{
|
||||||
perfData := &perftype.PerfData{Version: currentApiCallMetricsVersion}
|
|
||||||
item := perftype.DataItem{
|
|
||||||
Data: map[string]float64{
|
Data: map[string]float64{
|
||||||
"Perc50": float64(latency.Latency.Perc50) / 1000000, // us -> ms
|
"Perc50": float64(l.Perc50) / 1000000, // us -> ms
|
||||||
"Perc90": float64(latency.Latency.Perc90) / 1000000,
|
"Perc90": float64(l.Perc90) / 1000000,
|
||||||
"Perc99": float64(latency.Latency.Perc99) / 1000000,
|
"Perc99": float64(l.Perc99) / 1000000,
|
||||||
"Perc100": float64(latency.Latency.Perc100) / 1000000,
|
"Perc100": float64(l.Perc100) / 1000000,
|
||||||
},
|
},
|
||||||
Unit: "ms",
|
Unit: "ms",
|
||||||
Labels: map[string]string{
|
Labels: map[string]string{
|
||||||
"Metric": "pod_startup",
|
"Metric": name,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
perfData.DataItems = append(perfData.DataItems, item)
|
}
|
||||||
|
|
||||||
|
// PodStartupLatencyToPerfData transforms PodStartupLatency to PerfData.
|
||||||
|
func PodStartupLatencyToPerfData(latency *PodStartupLatency) *perftype.PerfData {
|
||||||
|
perfData := &perftype.PerfData{Version: currentApiCallMetricsVersion}
|
||||||
|
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.CreateToScheduleLatency, "create_to_schedule"))
|
||||||
|
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.ScheduleToRunLatency, "schedule_to_run"))
|
||||||
|
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.RunToWatchLatency, "run_to_watch"))
|
||||||
|
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.ScheduleToWatchLatency, "schedule_to_watch"))
|
||||||
|
perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.E2ELatency, "pod_startup"))
|
||||||
return perfData
|
return perfData
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -829,15 +829,23 @@ var _ = SIGDescribe("Density", func() {
|
|||||||
sort.Sort(framework.LatencySlice(schedToWatchLag))
|
sort.Sort(framework.LatencySlice(schedToWatchLag))
|
||||||
sort.Sort(framework.LatencySlice(e2eLag))
|
sort.Sort(framework.LatencySlice(e2eLag))
|
||||||
|
|
||||||
framework.PrintLatencies(scheduleLag, "worst schedule latencies")
|
framework.PrintLatencies(scheduleLag, "worst create-to-schedule latencies")
|
||||||
framework.PrintLatencies(startupLag, "worst run-after-schedule latencies")
|
framework.PrintLatencies(startupLag, "worst schedule-to-run latencies")
|
||||||
framework.PrintLatencies(watchLag, "worst watch latencies")
|
framework.PrintLatencies(watchLag, "worst run-to-watch latencies")
|
||||||
framework.PrintLatencies(schedToWatchLag, "worst scheduled-to-end total latencies")
|
framework.PrintLatencies(schedToWatchLag, "worst schedule-to-watch latencies")
|
||||||
framework.PrintLatencies(e2eLag, "worst e2e total latencies")
|
framework.PrintLatencies(e2eLag, "worst e2e latencies")
|
||||||
|
|
||||||
|
// Capture latency metrics related to pod-startup.
|
||||||
|
podStartupLatency := &framework.PodStartupLatency{
|
||||||
|
CreateToScheduleLatency: framework.ExtractLatencyMetrics(scheduleLag),
|
||||||
|
ScheduleToRunLatency: framework.ExtractLatencyMetrics(startupLag),
|
||||||
|
RunToWatchLatency: framework.ExtractLatencyMetrics(watchLag),
|
||||||
|
ScheduleToWatchLatency: framework.ExtractLatencyMetrics(schedToWatchLag),
|
||||||
|
E2ELatency: framework.ExtractLatencyMetrics(e2eLag),
|
||||||
|
}
|
||||||
|
f.TestSummaries = append(f.TestSummaries, podStartupLatency)
|
||||||
|
|
||||||
// Test whether e2e pod startup time is acceptable.
|
// Test whether e2e pod startup time is acceptable.
|
||||||
podStartupLatency := &framework.PodStartupLatency{Latency: framework.ExtractLatencyMetrics(e2eLag)}
|
|
||||||
f.TestSummaries = append(f.TestSummaries, podStartupLatency)
|
|
||||||
framework.ExpectNoError(framework.VerifyPodStartupLatency(podStartupLatency))
|
framework.ExpectNoError(framework.VerifyPodStartupLatency(podStartupLatency))
|
||||||
|
|
||||||
framework.LogSuspiciousLatency(startupLag, e2eLag, nodeCount, c)
|
framework.LogSuspiciousLatency(startupLag, e2eLag, nodeCount, c)
|
||||||
|
@ -556,14 +556,14 @@ func logAndVerifyLatency(batchLag time.Duration, e2eLags []framework.PodLatencyD
|
|||||||
latencyMetrics, _ := getPodStartLatency(kubeletAddr)
|
latencyMetrics, _ := getPodStartLatency(kubeletAddr)
|
||||||
framework.Logf("Kubelet Prometheus metrics (not reset):\n%s", framework.PrettyPrintJSON(latencyMetrics))
|
framework.Logf("Kubelet Prometheus metrics (not reset):\n%s", framework.PrettyPrintJSON(latencyMetrics))
|
||||||
|
|
||||||
podCreateLatency := framework.PodStartupLatency{Latency: framework.ExtractLatencyMetrics(e2eLags)}
|
podStartupLatency := framework.ExtractLatencyMetrics(e2eLags)
|
||||||
|
|
||||||
// log latency perf data
|
// log latency perf data
|
||||||
logPerfData(getLatencyPerfData(podCreateLatency.Latency, testInfo), "latency")
|
logPerfData(getLatencyPerfData(podStartupLatency, testInfo), "latency")
|
||||||
|
|
||||||
if isVerify {
|
if isVerify {
|
||||||
// check whether e2e pod startup time is acceptable.
|
// check whether e2e pod startup time is acceptable.
|
||||||
framework.ExpectNoError(verifyPodStartupLatency(podStartupLimits, podCreateLatency.Latency))
|
framework.ExpectNoError(verifyPodStartupLatency(podStartupLimits, podStartupLatency))
|
||||||
|
|
||||||
// check bactch pod creation latency
|
// check bactch pod creation latency
|
||||||
if podBatchStartupLimit > 0 {
|
if podBatchStartupLimit > 0 {
|
||||||
|
Loading…
Reference in New Issue
Block a user