diff --git a/pkg/kubelet/server/BUILD b/pkg/kubelet/server/BUILD index edfb943102f..fd735c0c4dd 100644 --- a/pkg/kubelet/server/BUILD +++ b/pkg/kubelet/server/BUILD @@ -23,6 +23,7 @@ go_library( "//pkg/kubelet/apis/resourcemetrics/v1alpha1:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/prober:go_default_library", + "//pkg/kubelet/server/metrics:go_default_library", "//pkg/kubelet/server/portforward:go_default_library", "//pkg/kubelet/server/remotecommand:go_default_library", "//pkg/kubelet/server/stats:go_default_library", @@ -105,6 +106,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//pkg/kubelet/server/metrics:all-srcs", "//pkg/kubelet/server/portforward:all-srcs", "//pkg/kubelet/server/remotecommand:all-srcs", "//pkg/kubelet/server/stats:all-srcs", diff --git a/pkg/kubelet/server/metrics/BUILD b/pkg/kubelet/server/metrics/BUILD new file mode 100644 index 00000000000..6d22a4d7fc6 --- /dev/null +++ b/pkg/kubelet/server/metrics/BUILD @@ -0,0 +1,28 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["metrics.go"], + importpath = "k8s.io/kubernetes/pkg/kubelet/server/metrics", + deps = [ + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/kubelet/server/metrics/metrics.go b/pkg/kubelet/server/metrics/metrics.go new file mode 100644 index 00000000000..48ad2d145d0 --- /dev/null +++ b/pkg/kubelet/server/metrics/metrics.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + kubeletSubsystem = "kubelet" +) + +var ( + // HTTPRequests tracks the number of the http requests received since the server started. + HTTPRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: kubeletSubsystem, + Name: "http_requests_total", + Help: "Number of the http requests received since the server started", + }, + // server_type aims to differentiate the readonly server and the readwrite server. + // long_running marks whether the request is long-running or not. + // Currently, long-running requests include exec/attach/portforward/debug. + []string{"method", "path", "host", "server_type", "long_running"}, + ) + // HTTPRequestsDuration tracks the duration in seconds to serve http requests. + HTTPRequestsDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: kubeletSubsystem, + Name: "http_requests_duration_seconds", + Help: "Duration in seconds to serve http requests", + // Use DefBuckets for now, will customize the buckets if necessary. + Buckets: prometheus.DefBuckets, + }, + []string{"method", "path", "host", "server_type", "long_running"}, + ) + // HTTPInflightRequests tracks the number of the inflight http requests. + HTTPInflightRequests = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: kubeletSubsystem, + Name: "http_inflight_requests", + Help: "Number of the inflight http requests", + }, + []string{"method", "path", "host", "server_type", "long_running"}, + ) +) + +var registerMetrics sync.Once + +// Register all metrics. +func Register() { + registerMetrics.Do(func() { + prometheus.MustRegister(HTTPRequests) + prometheus.MustRegister(HTTPRequestsDuration) + prometheus.MustRegister(HTTPInflightRequests) + }) +} + +// SinceInSeconds gets the time since the specified start in seconds. +func SinceInSeconds(start time.Time) float64 { + return time.Since(start).Seconds() +} diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index a4f75116558..4a9d9080d4c 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -63,6 +63,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/apis/resourcemetrics/v1alpha1" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/prober" + servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" "k8s.io/kubernetes/pkg/kubelet/server/portforward" remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" "k8s.io/kubernetes/pkg/kubelet/server/stats" @@ -807,6 +808,33 @@ func (s *Server) getPortForward(request *restful.Request, response *restful.Resp proxyStream(response.ResponseWriter, request.Request, url) } +// trimURLPath trims a URL path. +// For paths in the format of "/metrics/xxx", "metrics/xxx" is returned; +// For all other paths, the first part of the path is returned. +func trimURLPath(path string) string { + parts := strings.SplitN(strings.TrimPrefix(path, "/"), "/", 3) + if len(parts) == 0 { + return path + } + + if parts[0] == "metrics" && len(parts) > 1 { + return fmt.Sprintf("%s/%s", parts[0], parts[1]) + + } + return parts[0] +} + +// isLongRunningRequest determines whether the request is long-running or not. +func isLongRunningRequest(path string) bool { + longRunningRequestPaths := []string{"exec", "attach", "portforward", "debug"} + for _, p := range longRunningRequestPaths { + if p == path { + return true + } + } + return false +} + // ServeHTTP responds to HTTP requests on the Kubelet. func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { defer httplog.NewLogged(req, &w).StacktraceWhen( @@ -820,6 +848,27 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.StatusSwitchingProtocols, ), ).Log() + + // monitor http requests + var serverType string + if s.auth == nil { + serverType = "readonly" + } else { + serverType = "readwrite" + } + + method, path, host := req.Method, trimURLPath(req.URL.Path), req.URL.Host + + longRunning := strconv.FormatBool(isLongRunningRequest(path)) + + servermetrics.HTTPRequests.WithLabelValues(method, path, host, serverType, longRunning).Inc() + + servermetrics.HTTPInflightRequests.WithLabelValues(method, path, host, serverType, longRunning).Inc() + defer servermetrics.HTTPInflightRequests.WithLabelValues(method, path, host, serverType, longRunning).Dec() + + startTime := time.Now() + defer servermetrics.HTTPRequestsDuration.WithLabelValues(method, path, host, serverType, longRunning).Observe(servermetrics.SinceInSeconds(startTime)) + s.restfulCont.ServeHTTP(w, req) } diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index 1359eec86aa..7627b0bcc10 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -1666,3 +1666,24 @@ func TestDebuggingDisabledHandlers(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) } + +func TestTrimURLPath(t *testing.T) { + tests := []struct { + path, expected string + }{ + {"", ""}, + {"//", ""}, + {"/pods", "pods"}, + {"pods", "pods"}, + {"pods/", "pods"}, + {"good/", "good"}, + {"pods/probes", "pods"}, + {"metrics", "metrics"}, + {"metrics/resource", "metrics/resource"}, + {"metrics/hello", "metrics/hello"}, + } + + for _, test := range tests { + assert.Equal(t, test.expected, trimURLPath(test.path), fmt.Sprintf("path is: %s", test.path)) + } +}