diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 6fb4e1335f8..4c8a5215948 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -35,6 +35,8 @@ const ( DockerOperationsKey = "docker_operations_latency_microseconds" DockerErrorsKey = "docker_errors" PodWorkerStartLatencyKey = "pod_worker_start_latency_microseconds" + PLEGRelistLatencyKey = "pleg_relist_latency_microseconds" + PLEGRelistIntervalKey = "pleg_relist_interval_microseconds" ) var ( @@ -105,6 +107,20 @@ var ( }, []string{"operation_type"}, ) + PLEGRelistLatency = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: KubeletSubsystem, + Name: PLEGRelistLatencyKey, + Help: "Latency in microseconds for relisting pods in PLEG.", + }, + ) + PLEGRelistInterval = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: KubeletSubsystem, + Name: PLEGRelistIntervalKey, + Help: "Interval in microseconds between relisting in PLEG.", + }, + ) ) var registerMetrics sync.Once @@ -123,6 +139,8 @@ func Register(containerCache kubecontainer.RuntimeCache) { prometheus.MustRegister(ContainersPerPodCount) prometheus.MustRegister(DockerErrors) prometheus.MustRegister(newPodAndContainerCollector(containerCache)) + prometheus.MustRegister(PLEGRelistLatency) + prometheus.MustRegister(PLEGRelistInterval) }) } diff --git a/pkg/kubelet/pleg/generic.go b/pkg/kubelet/pleg/generic.go index 5d47500c258..7816062cf89 100644 --- a/pkg/kubelet/pleg/generic.go +++ b/pkg/kubelet/pleg/generic.go @@ -22,6 +22,7 @@ import ( "github.com/golang/glog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util" ) @@ -50,6 +51,8 @@ type GenericPLEG struct { eventChannel chan *PodLifecycleEvent // The internal cache for container information. containers map[string]containerInfo + // Time of the last relisting. + lastRelistTime time.Time } type containerInfo struct { @@ -101,6 +104,17 @@ func generateEvent(podID types.UID, cid string, oldState, newState kubecontainer // with the internal pods/containers, and generats events accordingly. func (g *GenericPLEG) relist() { glog.V(5).Infof("GenericPLEG: Relisting") + timestamp := time.Now() + + if !g.lastRelistTime.IsZero() { + metrics.PLEGRelistInterval.Observe(metrics.SinceInMicroseconds(g.lastRelistTime)) + } + defer func() { + // Update the relist time. + g.lastRelistTime = timestamp + metrics.PLEGRelistLatency.Observe(metrics.SinceInMicroseconds(timestamp)) + }() + // Get all the pods. pods, err := g.runtime.GetPods(true) if err != nil { diff --git a/pkg/metrics/kubelet_metrics.go b/pkg/metrics/kubelet_metrics.go index b9eb26717ff..66960dd75cf 100644 --- a/pkg/metrics/kubelet_metrics.go +++ b/pkg/metrics/kubelet_metrics.go @@ -74,6 +74,12 @@ var KnownKubeletMetrics = map[string][]string{ "kubelet_generate_pod_status_latency_microseconds": {"quantile"}, "kubelet_generate_pod_status_latency_microseconds_count": {}, "kubelet_generate_pod_status_latency_microseconds_sum": {}, + "kubelet_pleg_relist_latency_microseconds": {"quantile"}, + "kubelet_pleg_relist_latency_microseconds_sum": {}, + "kubelet_pleg_relist_latency_microseconds_count": {}, + "kubelet_pleg_relist_interval_microseconds": {"quantile"}, + "kubelet_pleg_relist_interval_microseconds_sum": {}, + "kubelet_pleg_relist_interval_microseconds_count": {}, "kubelet_pod_start_latency_microseconds": {"quantile"}, "kubelet_pod_start_latency_microseconds_count": {}, "kubelet_pod_start_latency_microseconds_sum": {},