From 11656478be93d4a9e54129ec35cd2b9558e901ac Mon Sep 17 00:00:00 2001 From: yue9944882 <291271447@qq.com> Date: Thu, 27 Feb 2020 17:04:17 +0800 Subject: [PATCH 1/3] register metrics from comp-base --- staging/src/k8s.io/apiserver/go.mod | 2 +- .../k8s.io/apiserver/pkg/server/filters/BUILD | 2 + .../server/filters/priority-and-fairness.go | 31 +++++++--- .../k8s.io/apiserver/pkg/server/routes/BUILD | 1 + .../apiserver/pkg/server/routes/metrics.go | 2 + .../pkg/util/flowcontrol/metrics/BUILD | 5 +- .../pkg/util/flowcontrol/metrics/metrics.go | 62 +++++++++++-------- 7 files changed, 71 insertions(+), 34 deletions(-) diff --git a/staging/src/k8s.io/apiserver/go.mod b/staging/src/k8s.io/apiserver/go.mod index 42383cc6419..b44cda39bed 100644 --- a/staging/src/k8s.io/apiserver/go.mod +++ b/staging/src/k8s.io/apiserver/go.mod @@ -29,12 +29,12 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 github.com/pkg/errors v0.8.1 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect - github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.2.0 github.com/sirupsen/logrus v1.4.2 // indirect github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 + go.uber.org/atomic v1.3.2 go.uber.org/zap v1.10.0 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD b/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD index d8fdedd70e0..3a52a3c53de 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD @@ -59,7 +59,9 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flowcontrol:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//vendor/go.uber.org/atomic:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go b/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go index e0f4fe7330f..d94d0301ad2 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go @@ -21,11 +21,9 @@ import ( "fmt" "net/http" - // TODO: decide whether to also generate the old metrics, which - // categorize according to mutating vs readonly. - - // "k8s.io/apiserver/pkg/endpoints/metrics" + _ "k8s.io/apiserver/pkg/util/flowcontrol/metrics" + "go.uber.org/atomic" fcv1a1 "k8s.io/api/flowcontrol/v1alpha1" apitypes "k8s.io/apimachinery/pkg/types" apirequest "k8s.io/apiserver/pkg/endpoints/request" @@ -38,8 +36,8 @@ type priorityAndFairnessKeyType int const priorityAndFairnessKey priorityAndFairnessKeyType = iota const ( - responseHeaderMatchedPriorityLevelConfigurationUID = "X-Kubernetes-PF-PriorityLevelUID" - responseHeaderMatchedFlowSchemaUID = "X-Kubernetes-PF-FlowSchemaUID" + responseHeaderMatchedPriorityLevelConfigurationUID = "X-Kubernetes-PF-PriorityLevel-UID" + responseHeaderMatchedFlowSchemaUID = "X-Kubernetes-PF-FlowSchema-UID" ) // PriorityAndFairnessClassification identifies the results of @@ -57,6 +55,8 @@ func GetClassification(ctx context.Context) *PriorityAndFairnessClassification { return ctx.Value(priorityAndFairnessKey).(*PriorityAndFairnessClassification) } +var atomicMutatingLen, atomicNonMutatingLen = atomic.NewInt32(0), atomic.NewInt32(0) + // WithPriorityAndFairness limits the number of in-flight // requests in a fine-grained way. func WithPriorityAndFairness( @@ -68,7 +68,7 @@ func WithPriorityAndFairness( klog.Warningf("priority and fairness support not found, skipping") return handler } - + startOnce.Do(startRecordingUsage) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() requestInfo, ok := apirequest.RequestInfoFrom(ctx) @@ -97,8 +97,25 @@ func WithPriorityAndFairness( PriorityLevelName: pl.Name, PriorityLevelUID: pl.UID} } + var served bool + isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb) execute := func() { + var mutatingLen, readOnlyLen int + if isMutatingRequest { + mutatingLen = int(atomicMutatingLen.Inc()) + } else { + readOnlyLen = int(atomicNonMutatingLen.Inc()) + } + defer func() { + if isMutatingRequest { + atomicMutatingLen.Dec() + watermark.recordMutating(mutatingLen) + } else { + atomicNonMutatingLen.Dec() + watermark.recordReadOnly(readOnlyLen) + } + }() served = true innerCtx := context.WithValue(ctx, priorityAndFairnessKey, classification) innerReq := r.Clone(innerCtx) diff --git a/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD b/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD index 57ac2c70f7e..bb97b8f5313 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/routes/BUILD @@ -26,6 +26,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics:go_default_library", "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go b/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go index 7b4591c8736..187deaded35 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go @@ -23,6 +23,7 @@ import ( apimetrics "k8s.io/apiserver/pkg/endpoints/metrics" "k8s.io/apiserver/pkg/server/mux" etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics" + flowcontrolmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -58,4 +59,5 @@ func (m MetricsWithReset) Install(c *mux.PathRecorderMux) { func register() { apimetrics.Register() etcd3metrics.Register() + flowcontrolmetrics.Register() } diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics/BUILD b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics/BUILD index 115224dc108..a1478b15992 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics/BUILD @@ -6,7 +6,10 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol/metrics", importpath = "k8s.io/apiserver/pkg/util/flowcontrol/metrics", visibility = ["//visibility:public"], - deps = ["//vendor/github.com/prometheus/client_golang/prometheus:go_default_library"], + deps = [ + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", + ], ) filegroup( diff --git a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics/metrics.go index d0280691202..887dd3db97c 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics/metrics.go @@ -17,9 +17,11 @@ limitations under the License. package metrics import ( + "sync" "time" - "github.com/prometheus/client_golang/prometheus" + compbasemetrics "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) const ( @@ -37,28 +39,29 @@ var ( requestDurationSecondsBuckets = []float64{0, 0.005, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30} ) -func init() { - prometheus.MustRegister(apiserverRejectedRequests) - prometheus.MustRegister(apiserverCurrentInqueueRequests) - prometheus.MustRegister(apiserverRequestQueueLength) - prometheus.MustRegister(apiserverRequestConcurrencyLimit) - prometheus.MustRegister(apiserverCurrentExecutingRequests) - prometheus.MustRegister(apiserverRequestWaitingSeconds) - prometheus.MustRegister(apiserverRequestExecutionSeconds) +var registerMetrics sync.Once + +// Register all metrics. +func Register() { + registerMetrics.Do(func() { + for _, metric := range metrics { + legacyregistry.MustRegister(metric) + } + }) } var ( - apiserverRejectedRequests = prometheus.NewCounterVec( - prometheus.CounterOpts{ + apiserverRejectedRequestsTotal = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "rejected_requests", + Name: "rejected_requests_total", Help: "Number of rejected requests by api priority and fairness system", }, []string{priorityLevel, "reason"}, ) - apiserverCurrentInqueueRequests = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + apiserverCurrentInqueueRequests = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, Name: "current_inqueue_requests", @@ -66,8 +69,8 @@ var ( }, []string{priorityLevel}, ) - apiserverRequestQueueLength = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + apiserverRequestQueueLength = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, Name: "request_queue_length", @@ -76,8 +79,8 @@ var ( }, []string{priorityLevel}, ) - apiserverRequestConcurrencyLimit = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + apiserverRequestConcurrencyLimit = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, Name: "request_concurrency_limit", @@ -85,8 +88,8 @@ var ( }, []string{priorityLevel}, ) - apiserverCurrentExecutingRequests = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + apiserverCurrentExecutingRequests = compbasemetrics.NewGaugeVec( + &compbasemetrics.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, Name: "current_executing_requests", @@ -94,8 +97,8 @@ var ( }, []string{priorityLevel}, ) - apiserverRequestWaitingSeconds = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + apiserverRequestWaitingSeconds = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, Name: "request_wait_duration_seconds", @@ -104,8 +107,8 @@ var ( }, []string{priorityLevel, flowSchema, "execute"}, ) - apiserverRequestExecutionSeconds = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + apiserverRequestExecutionSeconds = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, Name: "request_execution_seconds", @@ -114,6 +117,15 @@ var ( }, []string{priorityLevel, flowSchema}, ) + metrics = []compbasemetrics.Registerable{ + apiserverRejectedRequestsTotal, + apiserverCurrentInqueueRequests, + apiserverRequestQueueLength, + apiserverRequestConcurrencyLimit, + apiserverCurrentExecutingRequests, + apiserverRequestWaitingSeconds, + apiserverRequestExecutionSeconds, + } ) // UpdateFlowControlRequestsInQueue updates the value for the # of requests in the specified queues in flow control @@ -133,7 +145,7 @@ func UpdateSharedConcurrencyLimit(priorityLevel string, limit int) { // AddReject increments the # of rejected requests for flow control func AddReject(priorityLevel string, reason string) { - apiserverRejectedRequests.WithLabelValues(priorityLevel, reason).Add(1) + apiserverRejectedRequestsTotal.WithLabelValues(priorityLevel, reason).Add(1) } // ObserveQueueLength observes the queue length for flow control From dbe84361440697af5c53d12209524aad9068c81a Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Sun, 1 Mar 2020 18:10:20 -0500 Subject: [PATCH 2/3] Replaced uber atomic with sync atomic, removed unneded "blank import" --- .../src/k8s.io/apiserver/pkg/server/filters/BUILD | 2 -- .../pkg/server/filters/priority-and-fairness.go | 14 ++++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD b/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD index 3a52a3c53de..d8fdedd70e0 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD @@ -59,9 +59,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flowcontrol:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//vendor/go.uber.org/atomic:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go b/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go index d94d0301ad2..87bb76aa550 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go @@ -20,10 +20,8 @@ import ( "context" "fmt" "net/http" + "sync/atomic" - _ "k8s.io/apiserver/pkg/util/flowcontrol/metrics" - - "go.uber.org/atomic" fcv1a1 "k8s.io/api/flowcontrol/v1alpha1" apitypes "k8s.io/apimachinery/pkg/types" apirequest "k8s.io/apiserver/pkg/endpoints/request" @@ -55,7 +53,7 @@ func GetClassification(ctx context.Context) *PriorityAndFairnessClassification { return ctx.Value(priorityAndFairnessKey).(*PriorityAndFairnessClassification) } -var atomicMutatingLen, atomicNonMutatingLen = atomic.NewInt32(0), atomic.NewInt32(0) +var atomicMutatingLen, atomicNonMutatingLen int32 // WithPriorityAndFairness limits the number of in-flight // requests in a fine-grained way. @@ -103,16 +101,16 @@ func WithPriorityAndFairness( execute := func() { var mutatingLen, readOnlyLen int if isMutatingRequest { - mutatingLen = int(atomicMutatingLen.Inc()) + mutatingLen = int(atomic.AddInt32(&atomicMutatingLen, 1)) } else { - readOnlyLen = int(atomicNonMutatingLen.Inc()) + readOnlyLen = int(atomic.AddInt32(&atomicNonMutatingLen, 1)) } defer func() { if isMutatingRequest { - atomicMutatingLen.Dec() + atomic.AddInt32(&atomicMutatingLen, -11) watermark.recordMutating(mutatingLen) } else { - atomicNonMutatingLen.Dec() + atomic.AddInt32(&atomicNonMutatingLen, -1) watermark.recordReadOnly(readOnlyLen) } }() From 1cb42445d76daa79b2e3ec7f5a6ec0b32e994815 Mon Sep 17 00:00:00 2001 From: yue9944882 <291271447@qq.com> Date: Mon, 2 Mar 2020 21:44:58 +0800 Subject: [PATCH 3/3] hack/update-vendor.sh --- staging/src/k8s.io/apiserver/go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/staging/src/k8s.io/apiserver/go.mod b/staging/src/k8s.io/apiserver/go.mod index b44cda39bed..90996a0d481 100644 --- a/staging/src/k8s.io/apiserver/go.mod +++ b/staging/src/k8s.io/apiserver/go.mod @@ -34,7 +34,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 - go.uber.org/atomic v1.3.2 go.uber.org/zap v1.10.0 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9