Record event for lifecycle fallback to http

pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go

@@ -127,7 +127,8 @@ func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS
 	kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(
 		&fakeHTTP{},
 		kubeRuntimeManager,
-		kubeRuntimeManager)
+		kubeRuntimeManager,
+		recorder)
 
 	kubeRuntimeManager.getNodeAllocatable = func() v1.ResourceList {
 		return v1.ResourceList{

pkg/kubelet/kuberuntime/kuberuntime_container_test.go

@@ -295,7 +295,8 @@ func TestLifeCycleHook(t *testing.T) {
 	lcHanlder := lifecycle.NewHandlerRunner(
 		fakeHTTP,
 		fakeRunner,
-		fakePodStatusProvider)
+		fakePodStatusProvider,
+		nil)
 
 	m.runner = lcHanlder
 

pkg/kubelet/kuberuntime/kuberuntime_manager.go

@@ -265,7 +265,7 @@ func NewKubeGenericRuntimeManager(
 		serializeImagePulls,
 		imagePullQPS,
 		imagePullBurst)
-	kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(insecureContainerLifecycleHTTPClient, kubeRuntimeManager, kubeRuntimeManager)
+	kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(insecureContainerLifecycleHTTPClient, kubeRuntimeManager, kubeRuntimeManager, recorder)
 	kubeRuntimeManager.containerGC = newContainerGC(runtimeService, podStateProvider, kubeRuntimeManager)
 	kubeRuntimeManager.podStateProvider = podStateProvider
 

pkg/kubelet/lifecycle/handlers.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -48,6 +49,7 @@ type handlerRunner struct {
 	httpDoer         kubetypes.HTTPDoer
 	commandRunner    kubecontainer.CommandRunner
 	containerManager podStatusProvider
+	eventRecorder    record.EventRecorder
 }
 
 type podStatusProvider interface {
@@ -55,11 +57,12 @@ type podStatusProvider interface {
 }
 
 // NewHandlerRunner returns a configured lifecycle handler for a container.
-func NewHandlerRunner(httpDoer kubetypes.HTTPDoer, commandRunner kubecontainer.CommandRunner, containerManager podStatusProvider) kubecontainer.HandlerRunner {
+func NewHandlerRunner(httpDoer kubetypes.HTTPDoer, commandRunner kubecontainer.CommandRunner, containerManager podStatusProvider, eventRecorder record.EventRecorder) kubecontainer.HandlerRunner {
 	return &handlerRunner{
 		httpDoer:         httpDoer,
 		commandRunner:    commandRunner,
 		containerManager: containerManager,
+		eventRecorder:    eventRecorder,
 	}
 }
 
@@ -75,7 +78,7 @@ func (hr *handlerRunner) Run(containerID kubecontainer.ContainerID, pod *v1.Pod,
 		}
 		return msg, err
 	case handler.HTTPGet != nil:
-		err := hr.runHTTPHandler(pod, container, handler)
+		err := hr.runHTTPHandler(pod, container, handler, hr.eventRecorder)
 		var msg string
 		if err != nil {
 			msg = fmt.Sprintf("HTTP lifecycle hook (%s) for Container %q in Pod %q failed - error: %v", handler.HTTPGet.Path, container.Name, format.Pod(pod), err)
@@ -113,7 +116,7 @@ func resolvePort(portReference intstr.IntOrString, container *v1.Container) (int
 	return -1, fmt.Errorf("couldn't find port: %v in %v", portReference, container)
 }
 
-func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, handler *v1.LifecycleHandler) error {
+func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, handler *v1.LifecycleHandler, eventRecorder record.EventRecorder) error {
 	host := handler.HTTPGet.Host
 	podIP := host
 	if len(host) == 0 {
@@ -138,8 +141,6 @@ func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha
 		discardHTTPRespBody(resp)
 
 		if isHTTPResponseError(err) {
-			// TODO: emit an event about the fallback
-			// TODO: increment a metric about the fallback
 			klog.V(1).ErrorS(err, "HTTPS request to lifecycle hook got HTTP response, retrying with HTTP.", "pod", klog.KObj(pod), "host", req.URL.Host)
 
 			req := req.Clone(context.Background())
@@ -149,6 +150,11 @@ func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha
 
 			// clear err since the fallback succeeded
 			if httpErr == nil {
+				// TODO: increment a metric about the fallback
+				if eventRecorder != nil {
+					// report the fallback with an event
+					eventRecorder.Event(pod, v1.EventTypeWarning, "LifecycleHTTPFallback", fmt.Sprintf("request to HTTPS lifecycle hook %s got HTTP response, retry with HTTP succeeded", req.URL.Host))
+				}
 				err = nil
 			}
 			discardHTTPRespBody(resp)

pkg/kubelet/lifecycle/handlers_test.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/tools/record"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -115,7 +116,7 @@ func (f podStatusProviderFunc) GetPodStatus(uid types.UID, name, namespace strin
 
 func TestRunHandlerExec(t *testing.T) {
 	fakeCommandRunner := fakeContainerCommandRunner{}
-	handlerRunner := NewHandlerRunner(&fakeHTTP{}, &fakeCommandRunner, nil)
+	handlerRunner := NewHandlerRunner(&fakeHTTP{}, &fakeCommandRunner, nil, nil)
 
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
 	containerName := "containerFoo"
@@ -161,7 +162,7 @@ func (f *fakeHTTP) Do(req *http.Request) (*http.Response, error) {
 func TestRunHandlerHttp(t *testing.T) {
 	fakeHTTPGetter := fakeHTTP{}
 	fakePodStatusProvider := stubPodStatusProvider("127.0.0.1")
-	handlerRunner := NewHandlerRunner(&fakeHTTPGetter, &fakeContainerCommandRunner{}, fakePodStatusProvider)
+	handlerRunner := NewHandlerRunner(&fakeHTTPGetter, &fakeContainerCommandRunner{}, fakePodStatusProvider, nil)
 
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
 	containerName := "containerFoo"
@@ -197,7 +198,7 @@ func TestRunHandlerHttpWithHeaders(t *testing.T) {
 	fakeHTTPDoer := fakeHTTP{}
 	fakePodStatusProvider := stubPodStatusProvider("127.0.0.1")
 
-	handlerRunner := NewHandlerRunner(&fakeHTTPDoer, &fakeContainerCommandRunner{}, fakePodStatusProvider)
+	handlerRunner := NewHandlerRunner(&fakeHTTPDoer, &fakeContainerCommandRunner{}, fakePodStatusProvider, nil)
 
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
 	containerName := "containerFoo"
@@ -237,7 +238,7 @@ func TestRunHandlerHttpWithHeaders(t *testing.T) {
 func TestRunHandlerHttps(t *testing.T) {
 	fakeHTTPDoer := fakeHTTP{}
 	fakePodStatusProvider := stubPodStatusProvider("127.0.0.1")
-	handlerRunner := NewHandlerRunner(&fakeHTTPDoer, &fakeContainerCommandRunner{}, fakePodStatusProvider)
+	handlerRunner := NewHandlerRunner(&fakeHTTPDoer, &fakeContainerCommandRunner{}, fakePodStatusProvider, nil)
 
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
 	containerName := "containerFoo"
@@ -345,7 +346,7 @@ func TestRunHandlerHTTPPort(t *testing.T) {
 		t.Run(tt.Name, func(t *testing.T) {
 			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, tt.FeatureGateEnabled)()
 			fakeHTTPDoer := fakeHTTP{}
-			handlerRunner := NewHandlerRunner(&fakeHTTPDoer, &fakeContainerCommandRunner{}, fakePodStatusProvider)
+			handlerRunner := NewHandlerRunner(&fakeHTTPDoer, &fakeContainerCommandRunner{}, fakePodStatusProvider, nil)
 
 			container.Lifecycle.PostStart.HTTPGet.Port = tt.Port
 			pod.Spec.Containers = []v1.Container{container}
@@ -621,7 +622,7 @@ func TestRunHTTPHandler(t *testing.T) {
 
 			verify := func(t *testing.T, expectedHeader http.Header, expectedURL string) {
 				fakeHTTPDoer := fakeHTTP{}
-				handlerRunner := NewHandlerRunner(&fakeHTTPDoer, &fakeContainerCommandRunner{}, fakePodStatusProvider)
+				handlerRunner := NewHandlerRunner(&fakeHTTPDoer, &fakeContainerCommandRunner{}, fakePodStatusProvider, nil)
 
 				_, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
 				if err != nil {
@@ -650,7 +651,7 @@ func TestRunHTTPHandler(t *testing.T) {
 }
 
 func TestRunHandlerNil(t *testing.T) {
-	handlerRunner := NewHandlerRunner(&fakeHTTP{}, &fakeContainerCommandRunner{}, nil)
+	handlerRunner := NewHandlerRunner(&fakeHTTP{}, &fakeContainerCommandRunner{}, nil, nil)
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
 	podName := "podFoo"
 	podNamespace := "nsFoo"
@@ -675,7 +676,7 @@ func TestRunHandlerNil(t *testing.T) {
 func TestRunHandlerExecFailure(t *testing.T) {
 	expectedErr := fmt.Errorf("invalid command")
 	fakeCommandRunner := fakeContainerCommandRunner{Err: expectedErr, Msg: expectedErr.Error()}
-	handlerRunner := NewHandlerRunner(&fakeHTTP{}, &fakeCommandRunner, nil)
+	handlerRunner := NewHandlerRunner(&fakeHTTP{}, &fakeCommandRunner, nil, nil)
 
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
 	containerName := "containerFoo"
@@ -715,7 +716,7 @@ func TestRunHandlerHttpFailure(t *testing.T) {
 
 	fakePodStatusProvider := stubPodStatusProvider("127.0.0.1")
 
-	handlerRunner := NewHandlerRunner(&fakeHTTPGetter, &fakeContainerCommandRunner{}, fakePodStatusProvider)
+	handlerRunner := NewHandlerRunner(&fakeHTTPGetter, &fakeContainerCommandRunner{}, fakePodStatusProvider, nil)
 
 	containerName := "containerFoo"
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
@@ -759,9 +760,11 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	recorder := &record.FakeRecorder{Events: make(chan string, 10)}
+
 	fakePodStatusProvider := stubPodStatusProvider("127.0.0.1")
 
-	handlerRunner := NewHandlerRunner(srv.Client(), &fakeContainerCommandRunner{}, fakePodStatusProvider).(*handlerRunner)
+	handlerRunner := NewHandlerRunner(srv.Client(), &fakeContainerCommandRunner{}, fakePodStatusProvider, recorder).(*handlerRunner)
 
 	containerName := "containerFoo"
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
@@ -801,6 +804,15 @@ func TestRunHandlerHttpsFailureFallback(t *testing.T) {
 	if actualHeaders.Get("Authorization") != "" {
 		t.Error("unexpected Authorization header")
 	}
+
+	select {
+	case event := <-recorder.Events:
+		if !strings.Contains(event, "LifecycleHTTPFallback") {
+			t.Fatalf("expected LifecycleHTTPFallback event, got %q", event)
+		}
+	default:
+		t.Fatal("no event recorded")
+	}
 }
 
 func TestIsHTTPResponseError(t *testing.T) {