Merge pull request #111917 from logicalhan/request-sizes

Add request body size metric for the write path.

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/create.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
+	requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -86,7 +87,7 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
 			return
 		}
 
-		body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
+		body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Create)
 		trace.Step("limitedReadBody done", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
 		if err != nil {
 			scope.err(err, w, req)

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
+	requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -70,7 +71,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
 
 		options := &metav1.DeleteOptions{}
 		if allowsOptions {
-			body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
+			body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Patch)
 			if err != nil {
 				scope.err(err, w, req)
 				return

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/metrics/OWNERS

@@ -0,0 +1,4 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - logicalhan

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/metrics/metrics.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"context"
+	"k8s.io/component-base/metrics"
+)
+
+type RequestBodyVerb string
+
+const (
+	Patch  RequestBodyVerb = "patch"
+	Delete RequestBodyVerb = "delete"
+	Update RequestBodyVerb = "update"
+	Create RequestBodyVerb = "create"
+)
+
+var (
+	RequestBodySizes = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Subsystem: "apiserver",
+			Name:      "request_body_sizes",
+			Help:      "Apiserver request body sizes broken out by size.",
+			// we use 0.05 KB as the smallest bucket with 0.1 KB increments up to the
+			// apiserver limit.
+			Buckets:        metrics.LinearBuckets(50000, 100000, 31),
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"resource", "verb"},
+	)
+)
+
+func RecordRequestBodySize(ctx context.Context, resource string, verb RequestBodyVerb, size int) {
+	RequestBodySizes.WithContext(ctx).WithLabelValues(resource, string(verb)).Observe(float64(size))
+}

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go

@@ -23,9 +23,9 @@ import (
 	"strings"
 	"time"
 
+	jsonpatch "github.com/evanphx/json-patch"
 	kjson "sigs.k8s.io/json"
 
-	jsonpatch "github.com/evanphx/json-patch"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
@@ -45,6 +45,7 @@ import (
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
+	requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -98,7 +99,7 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
 			return
 		}
 
-		patchBytes, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
+		patchBytes, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Patch)
 		trace.Step("limitedReadBody done", utiltrace.Field{"len", len(patchBytes)}, utiltrace.Field{"err", err})
 		if err != nil {
 			scope.err(err, w, req)

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go

@@ -41,6 +41,7 @@ import (
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
+	requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/endpoints/request"
@@ -390,6 +391,15 @@ func limitedReadBody(req *http.Request, limit int64) ([]byte, error) {
 	return data, nil
 }
 
+func limitedReadBodyWithRecordMetric(ctx context.Context, req *http.Request, limit int64, resourceGroup string, verb requestmetrics.RequestBodyVerb) ([]byte, error) {
+	readBody, err := limitedReadBody(req, limit)
+	if err == nil {
+		// only record if we've read successfully
+		requestmetrics.RecordRequestBodySize(ctx, resourceGroup, verb, len(readBody))
+	}
+	return readBody, err
+}
+
 func isDryRun(url *url.URL) bool {
 	return len(url.Query()["dryRun"]) != 0
 }

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"reflect"
 	"strings"
@@ -45,9 +46,12 @@ import (
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/apis/example"
 	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
+	"k8s.io/apiserver/pkg/endpoints/handlers/metrics"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
 	utiltrace "k8s.io/utils/trace"
 )
 
@@ -107,6 +111,94 @@ func TestPatchAnonymousField(t *testing.T) {
 	}
 }
 
+func TestLimitedReadBody(t *testing.T) {
+	defer legacyregistry.Reset()
+	legacyregistry.Register(metrics.RequestBodySizes)
+
+	testcases := []struct {
+		desc            string
+		requestBody     io.Reader
+		limit           int64
+		expectedMetrics string
+		expectedErr     bool
+	}{
+		{
+			desc:            "aaaa with limit 1",
+			requestBody:     strings.NewReader("aaaa"),
+			limit:           1,
+			expectedMetrics: "",
+			expectedErr:     true,
+		},
+		{
+			desc:        "aaaa with limit 5",
+			requestBody: strings.NewReader("aaaa"),
+			limit:       5,
+			expectedMetrics: `
+        # HELP apiserver_request_body_sizes [ALPHA] Apiserver request body sizes broken out by size.
+        # TYPE apiserver_request_body_sizes histogram
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="50000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="150000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="250000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="350000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="450000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="550000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="650000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="750000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="850000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="950000"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.05e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.15e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.25e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.35e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.45e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.55e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.65e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.75e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.85e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.95e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.05e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.15e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.25e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.35e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.45e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.55e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.65e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.75e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.85e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.95e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="3.05e+06"} 1
+        apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="+Inf"} 1
+        apiserver_request_body_sizes_sum{resource="resource.group",verb="create"} 4
+        apiserver_request_body_sizes_count{resource="resource.group",verb="create"} 1
+`,
+			expectedErr: false,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.desc, func(t *testing.T) {
+			// reset metrics
+			defer metrics.RequestBodySizes.Reset()
+			defer legacyregistry.Reset()
+
+			req, err := http.NewRequest("POST", "/", tc.requestBody)
+			if err != nil {
+				t.Errorf("err not expected: got %v", err)
+			}
+			_, err = limitedReadBodyWithRecordMetric(context.Background(), req, tc.limit, "resource.group", metrics.Create)
+			if tc.expectedErr {
+				if err == nil {
+					t.Errorf("err expected: got nil")
+				}
+				return
+			}
+			if err = testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedMetrics), "apiserver_request_body_sizes"); err != nil {
+				t.Errorf("unexpected err: %v", err)
+			}
+		})
+	}
+}
+
 func TestStrategicMergePatchInvalid(t *testing.T) {
 	testGV := schema.GroupVersion{Group: "", Version: "v"}
 	scheme.AddKnownTypes(testGV, &testPatchType{})

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/update.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
+	requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -69,7 +70,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
 			return
 		}
 
-		body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
+		body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Update)
 		trace.Step("limitedReadBody done", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
 		if err != nil {
 			scope.err(err, w, req)

vendor/modules.txt

@@ -1565,6 +1565,7 @@ k8s.io/apiserver/pkg/endpoints/handlers
 k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager
 k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal
 k8s.io/apiserver/pkg/endpoints/handlers/finisher
+k8s.io/apiserver/pkg/endpoints/handlers/metrics
 k8s.io/apiserver/pkg/endpoints/handlers/negotiation
 k8s.io/apiserver/pkg/endpoints/handlers/responsewriters
 k8s.io/apiserver/pkg/endpoints/metrics