github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 21:47:07 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #58340 from gmarek/dropped

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 58446, 58459, 58340). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add apiserver metric for number of requests dropped by 'max-inflight-requests' filters.

Useful for figuring out on which dimension master is overloaded.

cc @sttts @lavalamp @deads2k @timothysc @hulkholden
This commit is contained in:
Kubernetes Submit Queue 2018-01-19 09:49:32 -08:00 committed by GitHub
commit 07ad1f7176
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go
View File

@ -78,9 +78,24 @@ var (
}, },
[]string{"verb", "resource", "subresource", "scope"}, []string{"verb", "resource", "subresource", "scope"},
) )
// DroppedRequests is a number of requests dropped with 'Try again later' reponse"
DroppedRequests = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "apiserver_dropped_requests",
Help: "Number of requests dropped with 'Try again later' reponse",
},
[]string{"requestKind"},
)
kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`) kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`)
) )
const (
// ReadOnlyKind is a string identifying read only request kind
ReadOnlyKind = "readOnly"
// MutatingKind is a string identifying mutating request kind
MutatingKind = "mutating"
)
func init() { func init() {
// Register all metrics. // Register all metrics.
prometheus.MustRegister(requestCounter) prometheus.MustRegister(requestCounter)
@ -88,6 +103,7 @@ func init() {
prometheus.MustRegister(requestLatencies) prometheus.MustRegister(requestLatencies)
prometheus.MustRegister(requestLatenciesSummary) prometheus.MustRegister(requestLatenciesSummary)
prometheus.MustRegister(responseSizes) prometheus.MustRegister(responseSizes)
prometheus.MustRegister(DroppedRequests)
} }
// Record records a single request to the standard metrics endpoints. For use by handlers that perform their own // Record records a single request to the standard metrics endpoints. For use by handlers that perform their own

9
staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go
View File

@ -79,7 +79,8 @@ func WithMaxInFlightLimit(
} }
var c chan bool var c chan bool
if !nonMutatingRequestVerbs.Has(requestInfo.Verb) { isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
if isMutatingRequest {
c = mutatingChan c = mutatingChan
} else { } else {
c = nonMutatingChan c = nonMutatingChan
@ -95,6 +96,12 @@ func WithMaxInFlightLimit(
handler.ServeHTTP(w, r) handler.ServeHTTP(w, r)
default: default:
// We need to split this data between buckets used for throttling.
if isMutatingRequest {
metrics.DroppedRequests.WithLabelValues(metrics.MutatingKind).Inc()
} else {
metrics.DroppedRequests.WithLabelValues(metrics.ReadOnlyKind).Inc()
}
// at this point we're about to return a 429, BUT not all actors should be rate limited. A system:master is so powerful // at this point we're about to return a 429, BUT not all actors should be rate limited. A system:master is so powerful
// that he should always get an answer. It's a super-admin or a loopback connection. // that he should always get an answer. It's a super-admin or a loopback connection.
if currUser, ok := apirequest.UserFrom(ctx); ok { if currUser, ok := apirequest.UserFrom(ctx); ok {