diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 6f1cd2803af..684c912f420 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -163,6 +163,14 @@ const ( // Special label for [DRAResourceClaimsInUseDesc] which counts ResourceClaims regardless of the driver. DRAResourceClaimsInUseAnyDriver = "" + + // Metric keys for in-place pod resize operations. + ContainerRequestedResizesKey = "container_requested_resizes_total" + PodResizeDurationMillisecondsKey = "pod_resize_duration_milliseconds" + PodPendingResizesKey = "pod_pending_resizes" + PodInfeasibleResizesKey = "pod_infeasible_resizes_total" + PodInProgressResizesKey = "pod_in_progress_resizes" + PodDeferredAcceptedResizesKey = "pod_deferred_accepted_resizes_total" ) type imageSizeBucket struct { @@ -194,10 +202,13 @@ var ( // The buckets max value 40 is based on the 45sec max gRPC timeout value defined // for the DRA gRPC calls in the pkg/kubelet/cm/dra/plugin/registration.go DRADurationBuckets = metrics.ExponentialBucketsRange(.1, 40, 15) + + // podResizeDurationBuckets is the bucket boundaries for pod_resize_duration_milliseconds metrics. + podResizeDurationBuckets = []float64{10, 50, 100, 500, 1000, 2000, 5000, 10000, 20000, 30000, 60000, 120000, 300000, 600000} ) var ( - // NodeName is a Gauge that tracks the ode's name. The count is always 1. + // NodeName is a Gauge that tracks the node's name. The count is always 1. NodeName = metrics.NewGaugeVec( &metrics.GaugeOpts{ Subsystem: KubeletSubsystem, @@ -1079,6 +1090,72 @@ var ( StabilityLevel: metrics.ALPHA, }, ) + + // ContainerRequestedResizes tracks the cumulative number of requested resizes at the container level. + ContainerRequestedResizes = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: KubeletSubsystem, + Name: ContainerRequestedResizesKey, + Help: "Number of requested resizes, counted at the container level. Different resources on the same container are counted separately. The 'requirement' label refers to 'memory' or 'limits'; the 'operation' label can be one of 'add', 'remove', 'increase' or 'decrease'.", + StabilityLevel: metrics.ALPHA, + }, + []string{"resource", "requirement", "operation"}, + ) + + // PodResizeDurationMilliseconds tracks the duration (in milliseconds) it takes to resize a pod. + PodResizeDurationMilliseconds = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: KubeletSubsystem, + Name: PodResizeDurationMillisecondsKey, + Help: "Duration in milliseconds to actuate a pod resize", + Buckets: podResizeDurationBuckets, + StabilityLevel: metrics.ALPHA, + }, + []string{"success"}, + ) + + // PodPendingResizes tracks the number of pending resizes for pods. + PodPendingResizes = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: PodPendingResizesKey, + Help: "Number of pending resizes for pods.", + StabilityLevel: metrics.ALPHA, + }, + []string{"reason"}, + ) + + // PodInfeasibleResizes tracks the number of infeasible resizes for pods. + PodInfeasibleResizes = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: KubeletSubsystem, + Name: PodInfeasibleResizesKey, + Help: "Number of infeasible resizes for pods.", + StabilityLevel: metrics.ALPHA, + }, + []string{"reason_detail"}, + ) + + // PodInProgressResizes tracks the number of in-progress resizes for pods. + PodInProgressResizes = metrics.NewGauge( + &metrics.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: PodInProgressResizesKey, + Help: "Number of in-progress resizes for pods.", + StabilityLevel: metrics.ALPHA, + }, + ) + + // PodDeferredAcceptedResizes tracks the cumulative number of deferred accepted resizes for pods. + PodDeferredAcceptedResizes = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: KubeletSubsystem, + Name: PodDeferredAcceptedResizesKey, + Help: "Cumulative number of resizes that were accepted after being deferred.", + StabilityLevel: metrics.ALPHA, + }, + []string{"retry_trigger"}, + ) ) var registerMetrics sync.Once @@ -1188,6 +1265,15 @@ func Register(collectors ...metrics.StableCollector) { legacyregistry.MustRegister(ImageVolumeMountedSucceedTotal) legacyregistry.MustRegister(ImageVolumeMountedErrorsTotal) } + + if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) { + legacyregistry.MustRegister(ContainerRequestedResizes) + legacyregistry.MustRegister(PodResizeDurationMilliseconds) + legacyregistry.MustRegister(PodPendingResizes) + legacyregistry.MustRegister(PodInfeasibleResizes) + legacyregistry.MustRegister(PodInProgressResizes) + legacyregistry.MustRegister(PodDeferredAcceptedResizes) + } }) }