From a5d785fae8ec5009c0f89a4763e241474baf51ea Mon Sep 17 00:00:00 2001
From: Jordan Liggitt <liggitt@google.com>
Date: Wed, 19 Oct 2022 14:38:57 -0400
Subject: [PATCH] Record metric for lifecycle fallback to http

---
 pkg/kubelet/lifecycle/handlers.go      |  3 ++-
 pkg/kubelet/lifecycle/handlers_test.go | 22 +++++++++++++++++++++-
 pkg/kubelet/metrics/metrics.go         | 12 ++++++++++++
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/pkg/kubelet/lifecycle/handlers.go b/pkg/kubelet/lifecycle/handlers.go
index 0736602f236..66bb834d3b3 100644
--- a/pkg/kubelet/lifecycle/handlers.go
+++ b/pkg/kubelet/lifecycle/handlers.go
@@ -35,6 +35,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/metrics"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	httpprobe "k8s.io/kubernetes/pkg/probe/http"
@@ -150,7 +151,7 @@ func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha
 
 			// clear err since the fallback succeeded
 			if httpErr == nil {
-				// TODO: increment a metric about the fallback
+				metrics.LifecycleHandlerHTTPFallbacks.Inc()
 				if eventRecorder != nil {
 					// report the fallback with an event
 					eventRecorder.Event(pod, v1.EventTypeWarning, "LifecycleHTTPFallback", fmt.Sprintf("request to HTTPS lifecycle hook %s got HTTP response, retry with HTTP succeeded", req.URL.Host))
diff --git a/pkg/kubelet/lifecycle/handlers_test.go b/pkg/kubelet/lifecycle/handlers_test.go
index 88f7d4a0b66..5d017b89df3 100644
--- a/pkg/kubelet/lifecycle/handlers_test.go
+++ b/pkg/kubelet/lifecycle/handlers_test.go
@@ -34,8 +34,11 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/record"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
 	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 )
 
@@ -750,6 +753,14 @@ func TestRunHandlerHttpFailure(t *testing.T) {
 }
 
 func TestRunHandlerHttpsFailureFallback(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
+
+	// Since prometheus' gatherer is global, other tests may have updated metrics already, so
+	// we need to reset them prior running this test.
+	// This also implies that we can't run this test in parallel with other tests.
+	metrics.Register()
+	legacyregistry.Reset()
+
 	var actualHeaders http.Header
 	srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
 		actualHeaders = r.Header.Clone()
@@ -792,7 +803,6 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
 	pod.ObjectMeta.Name = "podFoo"
 	pod.ObjectMeta.Namespace = "nsFoo"
 	pod.Spec.Containers = []v1.Container{container}
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
 	msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
 
 	if err != nil {
@@ -805,6 +815,16 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
 		t.Error("unexpected Authorization header")
 	}
 
+	expectedMetrics := `
+# HELP kubelet_lifecycle_handler_http_fallbacks_total [ALPHA] The number of times lifecycle handlers successfully fell back to http from https.
+# TYPE kubelet_lifecycle_handler_http_fallbacks_total counter
+kubelet_lifecycle_handler_http_fallbacks_total 1
+`
+
+	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedMetrics), "kubelet_lifecycle_handler_http_fallbacks_total"); err != nil {
+		t.Fatal(err)
+	}
+
 	select {
 	case event := <-recorder.Events:
 		if !strings.Contains(event, "LifecycleHTTPFallback") {
diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go
index 17267b888d7..bd296391b27 100644
--- a/pkg/kubelet/metrics/metrics.go
+++ b/pkg/kubelet/metrics/metrics.go
@@ -497,6 +497,15 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		},
 	)
+
+	LifecycleHandlerHTTPFallbacks = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Subsystem:      KubeletSubsystem,
+			Name:           "lifecycle_handler_http_fallbacks_total",
+			Help:           "The number of times lifecycle handlers successfully fell back to http from https.",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
 )
 
 var registerMetrics sync.Once
@@ -547,6 +556,9 @@ func Register(collectors ...metrics.StableCollector) {
 			legacyregistry.MustRegister(GracefulShutdownEndTime)
 		}
 
+		if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentHTTPGetHandlers) {
+			legacyregistry.MustRegister(LifecycleHandlerHTTPFallbacks)
+		}
 	})
 }