mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
Record metric for lifecycle fallback to http
This commit is contained in:
parent
122b43037e
commit
a5d785fae8
@ -35,6 +35,7 @@ import (
|
|||||||
"k8s.io/klog/v2"
|
"k8s.io/klog/v2"
|
||||||
"k8s.io/kubernetes/pkg/features"
|
"k8s.io/kubernetes/pkg/features"
|
||||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/metrics"
|
||||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
||||||
httpprobe "k8s.io/kubernetes/pkg/probe/http"
|
httpprobe "k8s.io/kubernetes/pkg/probe/http"
|
||||||
@ -150,7 +151,7 @@ func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha
|
|||||||
|
|
||||||
// clear err since the fallback succeeded
|
// clear err since the fallback succeeded
|
||||||
if httpErr == nil {
|
if httpErr == nil {
|
||||||
// TODO: increment a metric about the fallback
|
metrics.LifecycleHandlerHTTPFallbacks.Inc()
|
||||||
if eventRecorder != nil {
|
if eventRecorder != nil {
|
||||||
// report the fallback with an event
|
// report the fallback with an event
|
||||||
eventRecorder.Event(pod, v1.EventTypeWarning, "LifecycleHTTPFallback", fmt.Sprintf("request to HTTPS lifecycle hook %s got HTTP response, retry with HTTP succeeded", req.URL.Host))
|
eventRecorder.Event(pod, v1.EventTypeWarning, "LifecycleHTTPFallback", fmt.Sprintf("request to HTTPS lifecycle hook %s got HTTP response, retry with HTTP succeeded", req.URL.Host))
|
||||||
|
@ -34,8 +34,11 @@ import (
|
|||||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
"k8s.io/client-go/tools/record"
|
"k8s.io/client-go/tools/record"
|
||||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||||
|
"k8s.io/component-base/metrics/legacyregistry"
|
||||||
|
"k8s.io/component-base/metrics/testutil"
|
||||||
"k8s.io/kubernetes/pkg/features"
|
"k8s.io/kubernetes/pkg/features"
|
||||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/metrics"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -750,6 +753,14 @@ func TestRunHandlerHttpFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunHandlerHttpsFailureFallback(t *testing.T) {
|
func TestRunHandlerHttpsFailureFallback(t *testing.T) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
|
||||||
|
|
||||||
|
// Since prometheus' gatherer is global, other tests may have updated metrics already, so
|
||||||
|
// we need to reset them prior running this test.
|
||||||
|
// This also implies that we can't run this test in parallel with other tests.
|
||||||
|
metrics.Register()
|
||||||
|
legacyregistry.Reset()
|
||||||
|
|
||||||
var actualHeaders http.Header
|
var actualHeaders http.Header
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||||
actualHeaders = r.Header.Clone()
|
actualHeaders = r.Header.Clone()
|
||||||
@ -792,7 +803,6 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
|
|||||||
pod.ObjectMeta.Name = "podFoo"
|
pod.ObjectMeta.Name = "podFoo"
|
||||||
pod.ObjectMeta.Namespace = "nsFoo"
|
pod.ObjectMeta.Namespace = "nsFoo"
|
||||||
pod.Spec.Containers = []v1.Container{container}
|
pod.Spec.Containers = []v1.Container{container}
|
||||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
|
|
||||||
msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
|
msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -805,6 +815,16 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
|
|||||||
t.Error("unexpected Authorization header")
|
t.Error("unexpected Authorization header")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
expectedMetrics := `
|
||||||
|
# HELP kubelet_lifecycle_handler_http_fallbacks_total [ALPHA] The number of times lifecycle handlers successfully fell back to http from https.
|
||||||
|
# TYPE kubelet_lifecycle_handler_http_fallbacks_total counter
|
||||||
|
kubelet_lifecycle_handler_http_fallbacks_total 1
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedMetrics), "kubelet_lifecycle_handler_http_fallbacks_total"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case event := <-recorder.Events:
|
case event := <-recorder.Events:
|
||||||
if !strings.Contains(event, "LifecycleHTTPFallback") {
|
if !strings.Contains(event, "LifecycleHTTPFallback") {
|
||||||
|
@ -497,6 +497,15 @@ var (
|
|||||||
StabilityLevel: metrics.ALPHA,
|
StabilityLevel: metrics.ALPHA,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
LifecycleHandlerHTTPFallbacks = metrics.NewCounter(
|
||||||
|
&metrics.CounterOpts{
|
||||||
|
Subsystem: KubeletSubsystem,
|
||||||
|
Name: "lifecycle_handler_http_fallbacks_total",
|
||||||
|
Help: "The number of times lifecycle handlers successfully fell back to http from https.",
|
||||||
|
StabilityLevel: metrics.ALPHA,
|
||||||
|
},
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
var registerMetrics sync.Once
|
var registerMetrics sync.Once
|
||||||
@ -547,6 +556,9 @@ func Register(collectors ...metrics.StableCollector) {
|
|||||||
legacyregistry.MustRegister(GracefulShutdownEndTime)
|
legacyregistry.MustRegister(GracefulShutdownEndTime)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentHTTPGetHandlers) {
|
||||||
|
legacyregistry.MustRegister(LifecycleHandlerHTTPFallbacks)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user