From f5beb04c53ae46bd9212e4a4c97275f864c29caa Mon Sep 17 00:00:00 2001 From: Victor Marmol Date: Mon, 23 Feb 2015 15:42:04 -0800 Subject: [PATCH 1/2] Adding Prometheus /metrics handler. --- pkg/kubelet/server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/kubelet/server.go b/pkg/kubelet/server.go index 22a227269ac..8ffa0cb10f2 100644 --- a/pkg/kubelet/server.go +++ b/pkg/kubelet/server.go @@ -39,6 +39,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/util/httpstream/spdy" "github.com/golang/glog" "github.com/google/cadvisor/info" + "github.com/prometheus/client_golang/prometheus" ) // Server is a http.Handler which exposes kubelet functionality over HTTP. @@ -110,6 +111,7 @@ func (s *Server) InstallDebuggingHandlers() { s.mux.HandleFunc("/logs/", s.handleLogs) s.mux.HandleFunc("/containerLogs/", s.handleContainerLogs) + s.mux.Handle("/metrics", prometheus.Handler()) } // error serializes an error object into an HTTP response. From c40a60e9452438ef637bf214959835562568d980 Mon Sep 17 00:00:00 2001 From: Victor Marmol Date: Mon, 23 Feb 2015 16:22:12 -0800 Subject: [PATCH 2/2] Adding image pull latency metric. Also adding TODOs for other metrics. Part of #4604. --- pkg/kubelet/kubelet.go | 6 ++++ pkg/kubelet/metrics/metrics.go | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 pkg/kubelet/metrics/metrics.go diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 353ed4009bf..e06fc8edf84 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -38,6 +38,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/client/record" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/envvars" + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/metrics" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/volume" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" "github.com/GoogleCloudPlatform/kubernetes/pkg/probe" @@ -964,6 +965,11 @@ func (kl *Kubelet) createPodInfraContainer(pod *api.BoundPod) (dockertools.Docke } func (kl *Kubelet) pullImage(img string, ref *api.ObjectReference) error { + start := time.Now() + defer func() { + metrics.ImagePullLatency.Observe(float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())) + }() + if err := kl.dockerPuller.Pull(img); err != nil { if ref != nil { record.Eventf(ref, "failed", "Failed to pull image %q", img) diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go new file mode 100644 index 00000000000..8a5e409e757 --- /dev/null +++ b/pkg/kubelet/metrics/metrics.go @@ -0,0 +1,59 @@ +/* +Copyright 2015 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const kubeletSubsystem = "kubelet" + +var ( + ImagePullLatency = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: kubeletSubsystem, + Name: "image_pull_latency_microseconds", + Help: "Image pull latency in microseconds.", + }, + ) + // TODO(vmarmol): Implement. + // TODO(vmarmol): Split by source? + PodCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Subsystem: kubeletSubsystem, + Name: "pod_count", + Help: "Number of pods currently running.", + }, + ) + // TODO(vmarmol): Implement. + // TODO(vmarmol): Split by source? + ContainerCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Subsystem: kubeletSubsystem, + Name: "container_count", + Help: "Number of containers currently running.", + }, + ) + // TODO(vmarmol): Containers per pod + // TODO(vmarmol): Latency of pod startup + // TODO(vmarmol): Latency of SyncPods +) + +func init() { + // Register the metrics. + prometheus.MustRegister(ImagePullLatency) +}