Record metric for lifecycle fallback to http

This commit is contained in:
Jordan Liggitt 2022-10-19 14:38:57 -04:00
parent 122b43037e
commit a5d785fae8
No known key found for this signature in database
3 changed files with 35 additions and 2 deletions

View File

@ -35,6 +35,7 @@ import (
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/metrics"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/kubelet/util/format"
httpprobe "k8s.io/kubernetes/pkg/probe/http"
@ -150,7 +151,7 @@ func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha
// clear err since the fallback succeeded
if httpErr == nil {
// TODO: increment a metric about the fallback
metrics.LifecycleHandlerHTTPFallbacks.Inc()
if eventRecorder != nil {
// report the fallback with an event
eventRecorder.Event(pod, v1.EventTypeWarning, "LifecycleHTTPFallback", fmt.Sprintf("request to HTTPS lifecycle hook %s got HTTP response, retry with HTTP succeeded", req.URL.Host))

View File

@ -34,8 +34,11 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/tools/record"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/component-base/metrics/testutil"
"k8s.io/kubernetes/pkg/features"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/kubelet/util/format"
)
@ -750,6 +753,14 @@ func TestRunHandlerHttpFailure(t *testing.T) {
}
func TestRunHandlerHttpsFailureFallback(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
// Since prometheus' gatherer is global, other tests may have updated metrics already, so
// we need to reset them prior running this test.
// This also implies that we can't run this test in parallel with other tests.
metrics.Register()
legacyregistry.Reset()
var actualHeaders http.Header
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
actualHeaders = r.Header.Clone()
@ -792,7 +803,6 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
pod.ObjectMeta.Name = "podFoo"
pod.ObjectMeta.Namespace = "nsFoo"
pod.Spec.Containers = []v1.Container{container}
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
if err != nil {
@ -805,6 +815,16 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
t.Error("unexpected Authorization header")
}
expectedMetrics := `
# HELP kubelet_lifecycle_handler_http_fallbacks_total [ALPHA] The number of times lifecycle handlers successfully fell back to http from https.
# TYPE kubelet_lifecycle_handler_http_fallbacks_total counter
kubelet_lifecycle_handler_http_fallbacks_total 1
`
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedMetrics), "kubelet_lifecycle_handler_http_fallbacks_total"); err != nil {
t.Fatal(err)
}
select {
case event := <-recorder.Events:
if !strings.Contains(event, "LifecycleHTTPFallback") {

View File

@ -497,6 +497,15 @@ var (
StabilityLevel: metrics.ALPHA,
},
)
LifecycleHandlerHTTPFallbacks = metrics.NewCounter(
&metrics.CounterOpts{
Subsystem: KubeletSubsystem,
Name: "lifecycle_handler_http_fallbacks_total",
Help: "The number of times lifecycle handlers successfully fell back to http from https.",
StabilityLevel: metrics.ALPHA,
},
)
)
var registerMetrics sync.Once
@ -547,6 +556,9 @@ func Register(collectors ...metrics.StableCollector) {
legacyregistry.MustRegister(GracefulShutdownEndTime)
}
if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentHTTPGetHandlers) {
legacyregistry.MustRegister(LifecycleHandlerHTTPFallbacks)
}
})
}