diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 6543eafdadd..a3d59d50120 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -35,6 +35,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/meta" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/rest" + "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver/metrics" "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz" "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" @@ -47,46 +48,8 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -var ( - // TODO(a-robinson): Add unit tests for the handling of these metrics once - // the upstream library supports it. - requestCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "apiserver_request_count", - Help: "Counter of apiserver requests broken out for each verb, API resource, client, and HTTP response code.", - }, - []string{"verb", "resource", "client", "code"}, - ) - requestLatencies = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "apiserver_request_latencies", - Help: "Response latency distribution in microseconds for each verb, resource and client.", - // Use buckets ranging from 125 ms to 8 seconds. - Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7), - }, - []string{"verb", "resource"}, - ) - requestLatenciesSummary = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Name: "apiserver_request_latencies_summary", - Help: "Response latency summary in microseconds for each verb and resource.", - }, - []string{"verb", "resource"}, - ) -) - func init() { - prometheus.MustRegister(requestCounter) - prometheus.MustRegister(requestLatencies) - prometheus.MustRegister(requestLatenciesSummary) -} - -// monitor is a helper function for each HTTP request handler to use for -// instrumenting basic request counter and latency metrics. -func monitor(verb, resource *string, client string, httpCode *int, reqStart time.Time) { - requestCounter.WithLabelValues(*verb, *resource, client, strconv.Itoa(*httpCode)).Inc() - requestLatencies.WithLabelValues(*verb, *resource).Observe(float64((time.Since(reqStart)) / time.Microsecond)) - requestLatenciesSummary.WithLabelValues(*verb, *resource).Observe(float64((time.Since(reqStart)) / time.Microsecond)) + metrics.Register() } // monitorFilter creates a filter that reports the metrics for a given resource and action. @@ -95,7 +58,7 @@ func monitorFilter(action, resource string) restful.FilterFunction { reqStart := time.Now() chain.ProcessFilter(req, res) httpCode := res.StatusCode() - monitor(&action, &resource, util.GetClient(req.Request), &httpCode, reqStart) + metrics.Monitor(&action, &resource, util.GetClient(req.Request), &httpCode, reqStart) } } @@ -168,10 +131,13 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) error { // TODO: document all handlers // InstallSupport registers the APIServer support functions -func InstallSupport(mux Mux, ws *restful.WebService) { +func InstallSupport(mux Mux, ws *restful.WebService, enableResettingMetrics bool) { // TODO: convert healthz and metrics to restful and remove container arg healthz.InstallHandler(mux) mux.Handle("/metrics", prometheus.Handler()) + if enableResettingMetrics { + mux.HandleFunc("/resetMetrics", metrics.Reset) + } // Set up a service to return the git code version. ws.Path("/version") diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go index 7e2bc84172d..a9825cc97ef 100644 --- a/pkg/apiserver/apiserver_test.go +++ b/pkg/apiserver/apiserver_test.go @@ -230,7 +230,7 @@ func handleInternal(legacy bool, storage map[string]rest.Storage, admissionContr panic(fmt.Sprintf("unable to install container %s: %v", group.Version, err)) } ws := new(restful.WebService) - InstallSupport(mux, ws) + InstallSupport(mux, ws, false) container.Add(ws) return &defaultAPIServer{mux, group, container} } diff --git a/pkg/apiserver/metrics/metrics.go b/pkg/apiserver/metrics/metrics.go new file mode 100644 index 00000000000..6509480dd7a --- /dev/null +++ b/pkg/apiserver/metrics/metrics.go @@ -0,0 +1,74 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "io" + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + // TODO(a-robinson): Add unit tests for the handling of these metrics once + // the upstream library supports it. + requestCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "apiserver_request_count", + Help: "Counter of apiserver requests broken out for each verb, API resource, client, and HTTP response code.", + }, + []string{"verb", "resource", "client", "code"}, + ) + requestLatencies = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "apiserver_request_latencies", + Help: "Response latency distribution in microseconds for each verb, resource and client.", + // Use buckets ranging from 125 ms to 8 seconds. + Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7), + }, + []string{"verb", "resource"}, + ) + requestLatenciesSummary = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "apiserver_request_latencies_summary", + Help: "Response latency summary in microseconds for each verb and resource.", + }, + []string{"verb", "resource"}, + ) +) + +// Register all metrics. +func Register() { + prometheus.MustRegister(requestCounter) + prometheus.MustRegister(requestLatencies) + prometheus.MustRegister(requestLatenciesSummary) +} + +func Monitor(verb, resource *string, client string, httpCode *int, reqStart time.Time) { + requestCounter.WithLabelValues(*verb, *resource, client, strconv.Itoa(*httpCode)).Inc() + requestLatencies.WithLabelValues(*verb, *resource).Observe(float64((time.Since(reqStart)) / time.Microsecond)) + requestLatenciesSummary.WithLabelValues(*verb, *resource).Observe(float64((time.Since(reqStart)) / time.Microsecond)) +} + +func Reset(w http.ResponseWriter, req *http.Request) { + requestCounter.Reset() + requestLatencies.Reset() + requestLatenciesSummary.Reset() + io.WriteString(w, "metrics reset\n") +} diff --git a/pkg/apiserver/proxy.go b/pkg/apiserver/proxy.go index d2b90358574..e04b2a6478a 100644 --- a/pkg/apiserver/proxy.go +++ b/pkg/apiserver/proxy.go @@ -31,6 +31,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/rest" + "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver/metrics" "github.com/GoogleCloudPlatform/kubernetes/pkg/httplog" "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" @@ -58,7 +59,7 @@ func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { var apiResource string var httpCode int reqStart := time.Now() - defer monitor(&verb, &apiResource, util.GetClient(req), &httpCode, reqStart) + defer metrics.Monitor(&verb, &apiResource, util.GetClient(req), &httpCode, reqStart) requestInfo, err := r.apiRequestInfoResolver.GetAPIRequestInfo(req) if err != nil { diff --git a/pkg/master/master.go b/pkg/master/master.go index 5e491ee418d..26c549c6973 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -552,7 +552,7 @@ func (m *Master) init(c *Config) { apiVersions = append(apiVersions, "v1") } - apiserver.InstallSupport(m.muxHelper, m.rootWebService) + apiserver.InstallSupport(m.muxHelper, m.rootWebService, c.EnableProfiling) apiserver.AddApiWebService(m.handlerContainer, c.APIPrefix, apiVersions) defaultVersion := m.defaultAPIGroupVersion() requestInfoResolver := &apiserver.APIRequestInfoResolver{util.NewStringSet(strings.TrimPrefix(defaultVersion.Root, "/")), defaultVersion.Mapper}