Merge pull request #113682 from pawbana/add-verb-resource-to-api-server-tracing

Add verb resource to api server tracing

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/helpers.go

@@ -21,6 +21,8 @@ import (
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 )
 
 const (
@@ -88,3 +90,34 @@ func (lazy *lazyAuditID) String() string {
 
 	return "unknown"
 }
+
+// lazyVerb implements String() string and it will
+// lazily get normalized Verb
+type lazyVerb struct {
+	req *http.Request
+}
+
+func (lazy *lazyVerb) String() string {
+	if lazy.req == nil {
+		return "unknown"
+	}
+	return metrics.NormalizedVerb(lazy.req)
+}
+
+// lazyResource implements String() string and it will
+// lazily get Resource from request info
+type lazyResource struct {
+	req *http.Request
+}
+
+func (lazy *lazyResource) String() string {
+	if lazy.req != nil {
+		ctx := lazy.req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if ok {
+			return requestInfo.Resource
+		}
+	}
+
+	return "unknown"
+}

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/helpers_test.go

@@ -17,10 +17,14 @@ limitations under the License.
 package handlers
 
 import (
+	"context"
 	"fmt"
-	"github.com/stretchr/testify/assert"
 	"net/http"
+	"net/url"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apiserver/pkg/endpoints/request"
 )
 
 func TestLazyTruncatedUserAgent(t *testing.T) {
@@ -71,3 +75,24 @@ func TestLazyAccept(t *testing.T) {
 	acceptWithoutReq := &lazyAccept{}
 	assert.Equal(t, "unknown", fmt.Sprintf("%v", acceptWithoutReq))
 }
+
+func TestLazyVerb(t *testing.T) {
+	assert.Equal(t, "unknown", fmt.Sprintf("%v", &lazyVerb{}))
+
+	u, _ := url.Parse("?watch=true")
+	req := &http.Request{Method: "GET", URL: u}
+	verbWithReq := &lazyVerb{req: req}
+	assert.Equal(t, "WATCH", fmt.Sprintf("%v", verbWithReq))
+}
+
+func TestLazyResource(t *testing.T) {
+	assert.Equal(t, "unknown", fmt.Sprintf("%v", &lazyResource{}))
+
+	resourceWithEmptyReq := &lazyResource{&http.Request{}}
+	assert.Equal(t, "unknown", fmt.Sprintf("%v", resourceWithEmptyReq))
+
+	req := &http.Request{}
+	ctx := request.WithRequestInfo(context.TODO(), &request.RequestInfo{Resource: "resource"})
+	resourceWithReq := &lazyResource{req: req.WithContext(ctx)}
+	assert.Equal(t, "resource", fmt.Sprintf("%v", resourceWithReq))
+}

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/trace_util.go

@@ -24,11 +24,13 @@ import (
 
 func traceFields(req *http.Request) []attribute.KeyValue {
 	return []attribute.KeyValue{
-		attribute.String("url", req.URL.Path),
+		attribute.Stringer("accept", &lazyAccept{req: req}),
-		attribute.Stringer("user-agent", &lazyTruncatedUserAgent{req: req}),
 		attribute.Stringer("audit-id", &lazyAuditID{req: req}),
 		attribute.Stringer("client", &lazyClientIP{req: req}),
-		attribute.Stringer("accept", &lazyAccept{req: req}),
 		attribute.String("protocol", req.Proto),
+		attribute.Stringer("resource", &lazyResource{req: req}),
+		attribute.String("url", req.URL.Path),
+		attribute.Stringer("user-agent", &lazyTruncatedUserAgent{req: req}),
+		attribute.Stringer("verb", &lazyVerb{req: req}),
 	}
 }

staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go

@@ -565,6 +565,20 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c
 	}
 }
 
+// NormalizedVerb returns normalized verb
+func NormalizedVerb(req *http.Request) string {
+	verb := req.Method
+	if requestInfo, ok := request.RequestInfoFrom(req.Context()); ok {
+		// If we can find a requestInfo, we can get a scope, and then
+		// we can convert GETs to LISTs when needed.
+		scope := CleanScope(requestInfo)
+		verb = CanonicalVerb(strings.ToUpper(verb), scope)
+	}
+
+	// mark APPLY requests and WATCH requests correctly.
+	return CleanVerb(verb, req)
+}
+
 // CleanScope returns the scope of the request.
 func CleanScope(requestInfo *request.RequestInfo) string {
 	if requestInfo.Name != "" || requestInfo.Verb == "create" {

staging/src/k8s.io/apiserver/pkg/server/httplog/httplog.go

@@ -243,16 +243,7 @@ func SetStacktracePredicate(ctx context.Context, pred StacktracePred) {
 func (rl *respLogger) Log() {
 	latency := time.Since(rl.startTime)
 	auditID := audit.GetAuditIDTruncated(rl.req.Context())
-
+	verb := metrics.NormalizedVerb(rl.req)
-	verb := rl.req.Method
-	if requestInfo, ok := request.RequestInfoFrom(rl.req.Context()); ok {
-		// If we can find a requestInfo, we can get a scope, and then
-		// we can convert GETs to LISTs when needed.
-		scope := metrics.CleanScope(requestInfo)
-		verb = metrics.CanonicalVerb(strings.ToUpper(verb), scope)
-	}
-	// mark APPLY requests and WATCH requests correctly.
-	verb = metrics.CleanVerb(verb, rl.req)
 
 	keysAndValues := []interface{}{
 		"verb", verb,