diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 48b7beef00c..3806c47f658 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -18,11 +18,12 @@ package metrics import ( "fmt" - "k8s.io/component-base/metrics" - "k8s.io/component-base/metrics/legacyregistry" "sync" "time" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -45,6 +46,7 @@ const ( PLEGRelistIntervalKey = "pleg_relist_interval_seconds" EvictionsKey = "evictions" EvictionStatsAgeKey = "eviction_stats_age_seconds" + PreemptionsKey = "preemptions" DeprecatedPodWorkerLatencyKey = "pod_worker_latency_microseconds" DeprecatedPodStartLatencyKey = "pod_start_latency_microseconds" DeprecatedCgroupManagerOperationsKey = "cgroup_manager_latency_microseconds" @@ -242,6 +244,18 @@ var ( }, []string{"eviction_signal"}, ) + // Preemptions is a Counter that tracks the cumulative number of pod preemptions initiated by the kubelet. + // Broken down by preemption signal. A preemption is only recorded for one resource, the sum of all signals + // is the number of preemptions on the given node. + Preemptions = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: KubeletSubsystem, + Name: PreemptionsKey, + Help: "Cumulative number of pod preemptions by preemption resource", + StabilityLevel: metrics.ALPHA, + }, + []string{"preemption_signal"}, + ) // DevicePluginRegistrationCount is a Counter that tracks the cumulative number of device plugin registrations. // Broken down by resource name. DevicePluginRegistrationCount = metrics.NewCounterVec( @@ -502,6 +516,7 @@ func Register(containerCache kubecontainer.RuntimeCache, collectors ...metrics.C legacyregistry.MustRegister(RuntimeOperationsErrors) legacyregistry.MustRegister(Evictions) legacyregistry.MustRegister(EvictionStatsAge) + legacyregistry.MustRegister(Preemptions) legacyregistry.MustRegister(DevicePluginRegistrationCount) legacyregistry.MustRegister(DevicePluginAllocationDuration) legacyregistry.MustRegister(DeprecatedPodWorkerLatency) diff --git a/pkg/kubelet/preemption/BUILD b/pkg/kubelet/preemption/BUILD index 7869ab83683..9be67a0ff67 100644 --- a/pkg/kubelet/preemption/BUILD +++ b/pkg/kubelet/preemption/BUILD @@ -16,6 +16,7 @@ go_library( "//pkg/kubelet/events:go_default_library", "//pkg/kubelet/eviction:go_default_library", "//pkg/kubelet/lifecycle:go_default_library", + "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/format:go_default_library", "//pkg/scheduler/algorithm/predicates:go_default_library", diff --git a/pkg/kubelet/preemption/preemption.go b/pkg/kubelet/preemption/preemption.go index 86e3333a68c..81eeff357bb 100644 --- a/pkg/kubelet/preemption/preemption.go +++ b/pkg/kubelet/preemption/preemption.go @@ -28,6 +28,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/lifecycle" + "k8s.io/kubernetes/pkg/kubelet/metrics" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" @@ -111,6 +112,11 @@ func (c *CriticalPodAdmissionHandler) evictPodsToFreeRequests(admitPod *v1.Pod, // In future syncPod loops, the kubelet will retry the pod deletion steps that it was stuck on. continue } + if len(insufficientResources) > 0 { + metrics.Preemptions.WithLabelValues(insufficientResources[0].resourceName.String()).Inc() + } else { + metrics.Preemptions.WithLabelValues("").Inc() + } klog.Infof("preemption: pod %s evicted successfully", format.Pod(pod)) } return nil