From 9dadafa8979619495317f166bf8bd4803ec89452 Mon Sep 17 00:00:00 2001
From: Jonathan Tomer <jktomer@google.com>
Date: Mon, 7 Oct 2019 15:21:17 -0700
Subject: [PATCH] healthz: instrument root healthz requests for metrics

---
 .../k8s.io/apiserver/pkg/server/healthz/BUILD |  4 +++
 .../apiserver/pkg/server/healthz/healthz.go   | 11 ++++++-
 .../pkg/server/healthz/healthz_test.go        | 33 +++++++++++++++++++
 3 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD b/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD
index 64ffaa25d13..8ed8903ba54 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD
+++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD
@@ -12,6 +12,9 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library",
+        "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
+        "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
     ],
 )
 
@@ -26,6 +29,7 @@ go_library(
     deps = [
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],
diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go
index a55e201e704..b5ca8afb294 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go
+++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go
@@ -27,6 +27,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/server/httplog"
 	"k8s.io/klog"
 )
@@ -122,7 +123,15 @@ func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
 
 	klog.V(5).Infof("Installing health checkers for (%v): %v", path, formatQuoted(checkerNames(checks...)...))
 
-	mux.Handle(path, handleRootHealthz(checks...))
+	mux.Handle(path,
+		metrics.InstrumentHandlerFunc("GET",
+			/* group = */ "",
+			/* version = */ "",
+			/* resource = */ "",
+			/* subresource = */ path,
+			/* scope = */ "",
+			/* component = */ "",
+			handleRootHealthz(checks...)))
 	for _, check := range checks {
 		mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check))
 	}
diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go
index aae39e0c87c..ede180c88f7 100644
--- a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go
@@ -23,9 +23,13 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"reflect"
+	"strings"
 	"testing"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
 )
 
 func TestInstallHandler(t *testing.T) {
@@ -232,6 +236,35 @@ func TestGetExcludedChecks(t *testing.T) {
 	}
 }
 
+func TestMetrics(t *testing.T) {
+	mux := http.NewServeMux()
+	InstallHandler(mux)
+	InstallLivezHandler(mux)
+	InstallReadyzHandler(mux)
+	metrics.Register()
+	metrics.Reset()
+
+	paths := []string{"/healthz", "/livez", "/readyz"}
+	for _, path := range paths {
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://example.com%s", path), nil)
+		if err != nil {
+			t.Errorf("%v", err)
+		}
+		mux.ServeHTTP(httptest.NewRecorder(), req)
+	}
+
+	expected := strings.NewReader(`
+        # HELP apiserver_request_total [ALPHA] Counter of apiserver requests broken out for each verb, dry run value, group, version, resource, scope, component, client, and HTTP response contentType and code.
+        # TYPE apiserver_request_total counter
+        apiserver_request_total{client="unknown",code="200",component="",contentType="text/plain; charset=utf-8",dry_run="",group="",resource="",scope="",subresource="/healthz",verb="GET",version=""} 1
+        apiserver_request_total{client="unknown",code="200",component="",contentType="text/plain; charset=utf-8",dry_run="",group="",resource="",scope="",subresource="/livez",verb="GET",version=""} 1
+        apiserver_request_total{client="unknown",code="200",component="",contentType="text/plain; charset=utf-8",dry_run="",group="",resource="",scope="",subresource="/readyz",verb="GET",version=""} 1
+`)
+	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, expected, "apiserver_request_total"); err != nil {
+		t.Error(err)
+	}
+}
+
 func createGetRequestWithUrl(rawUrlString string) *http.Request {
 	url, _ := url.Parse(rawUrlString)
 	return &http.Request{