mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 03:11:40 +00:00
Record metric for lifecycle fallback to http
This commit is contained in:
parent
122b43037e
commit
a5d785fae8
@ -35,6 +35,7 @@ import (
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
"k8s.io/kubernetes/pkg/kubelet/metrics"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
||||
httpprobe "k8s.io/kubernetes/pkg/probe/http"
|
||||
@ -150,7 +151,7 @@ func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha
|
||||
|
||||
// clear err since the fallback succeeded
|
||||
if httpErr == nil {
|
||||
// TODO: increment a metric about the fallback
|
||||
metrics.LifecycleHandlerHTTPFallbacks.Inc()
|
||||
if eventRecorder != nil {
|
||||
// report the fallback with an event
|
||||
eventRecorder.Event(pod, v1.EventTypeWarning, "LifecycleHTTPFallback", fmt.Sprintf("request to HTTPS lifecycle hook %s got HTTP response, retry with HTTP succeeded", req.URL.Host))
|
||||
|
@ -34,8 +34,11 @@ import (
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/client-go/tools/record"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
"k8s.io/component-base/metrics/testutil"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
"k8s.io/kubernetes/pkg/kubelet/metrics"
|
||||
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
||||
)
|
||||
|
||||
@ -750,6 +753,14 @@ func TestRunHandlerHttpFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunHandlerHttpsFailureFallback(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
|
||||
|
||||
// Since prometheus' gatherer is global, other tests may have updated metrics already, so
|
||||
// we need to reset them prior running this test.
|
||||
// This also implies that we can't run this test in parallel with other tests.
|
||||
metrics.Register()
|
||||
legacyregistry.Reset()
|
||||
|
||||
var actualHeaders http.Header
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||
actualHeaders = r.Header.Clone()
|
||||
@ -792,7 +803,6 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
|
||||
pod.ObjectMeta.Name = "podFoo"
|
||||
pod.ObjectMeta.Namespace = "nsFoo"
|
||||
pod.Spec.Containers = []v1.Container{container}
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
|
||||
msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
|
||||
|
||||
if err != nil {
|
||||
@ -805,6 +815,16 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
|
||||
t.Error("unexpected Authorization header")
|
||||
}
|
||||
|
||||
expectedMetrics := `
|
||||
# HELP kubelet_lifecycle_handler_http_fallbacks_total [ALPHA] The number of times lifecycle handlers successfully fell back to http from https.
|
||||
# TYPE kubelet_lifecycle_handler_http_fallbacks_total counter
|
||||
kubelet_lifecycle_handler_http_fallbacks_total 1
|
||||
`
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedMetrics), "kubelet_lifecycle_handler_http_fallbacks_total"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case event := <-recorder.Events:
|
||||
if !strings.Contains(event, "LifecycleHTTPFallback") {
|
||||
|
@ -497,6 +497,15 @@ var (
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
)
|
||||
|
||||
LifecycleHandlerHTTPFallbacks = metrics.NewCounter(
|
||||
&metrics.CounterOpts{
|
||||
Subsystem: KubeletSubsystem,
|
||||
Name: "lifecycle_handler_http_fallbacks_total",
|
||||
Help: "The number of times lifecycle handlers successfully fell back to http from https.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
var registerMetrics sync.Once
|
||||
@ -547,6 +556,9 @@ func Register(collectors ...metrics.StableCollector) {
|
||||
legacyregistry.MustRegister(GracefulShutdownEndTime)
|
||||
}
|
||||
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentHTTPGetHandlers) {
|
||||
legacyregistry.MustRegister(LifecycleHandlerHTTPFallbacks)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user