diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go index 8a3b20d0d86..ec75fad3d12 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go @@ -78,9 +78,24 @@ var ( }, []string{"verb", "resource", "subresource", "scope"}, ) + // DroppedRequests is a number of requests dropped with 'Try again later' reponse" + DroppedRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "apiserver_dropped_requests", + Help: "Number of requests dropped with 'Try again later' reponse", + }, + []string{"requestKind"}, + ) kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`) ) +const ( + // ReadOnlyKind is a string identifying read only request kind + ReadOnlyKind = "readOnly" + // MutatingKind is a string identifying mutating request kind + MutatingKind = "mutating" +) + func init() { // Register all metrics. prometheus.MustRegister(requestCounter) @@ -88,6 +103,7 @@ func init() { prometheus.MustRegister(requestLatencies) prometheus.MustRegister(requestLatenciesSummary) prometheus.MustRegister(responseSizes) + prometheus.MustRegister(DroppedRequests) } // Record records a single request to the standard metrics endpoints. For use by handlers that perform their own diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go b/staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go index 47ed949131d..1eb7292a4f3 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go @@ -79,7 +79,8 @@ func WithMaxInFlightLimit( } var c chan bool - if !nonMutatingRequestVerbs.Has(requestInfo.Verb) { + isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb) + if isMutatingRequest { c = mutatingChan } else { c = nonMutatingChan @@ -95,6 +96,12 @@ func WithMaxInFlightLimit( handler.ServeHTTP(w, r) default: + // We need to split this data between buckets used for throttling. + if isMutatingRequest { + metrics.DroppedRequests.WithLabelValues(metrics.MutatingKind).Inc() + } else { + metrics.DroppedRequests.WithLabelValues(metrics.ReadOnlyKind).Inc() + } // at this point we're about to return a 429, BUT not all actors should be rate limited. A system:master is so powerful // that he should always get an answer. It's a super-admin or a loopback connection. if currUser, ok := apirequest.UserFrom(ctx); ok {