fallback to http when lifecycle handler request should have been https

pkg/kubelet/lifecycle/handlers.go

@@ -17,11 +17,15 @@ limitations under the License.
 package lifecycle
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"strconv"
+	"strings"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -132,6 +136,23 @@ func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha
 		}
 		resp, err := hr.httpDoer.Do(req)
 		discardHTTPRespBody(resp)
+
+		if isHTTPResponseError(err) {
+			// TODO: emit an event about the fallback
+			// TODO: increment a metric about the fallback
+			klog.V(1).ErrorS(err, "HTTPS request to lifecycle hook got HTTP response, retrying with HTTP.", "pod", klog.KObj(pod), "host", req.URL.Host)
+
+			req := req.Clone(context.Background())
+			req.URL.Scheme = "http"
+			req.Header.Del("Authorization")
+			resp, httpErr := hr.httpDoer.Do(req)
+
+			// clear err since the fallback succeeded
+			if httpErr == nil {
+				err = nil
+			}
+			discardHTTPRespBody(resp)
+		}
 		return err
 	}
 
@@ -201,3 +222,14 @@ func (a *appArmorAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult {
 		Message: fmt.Sprintf("Cannot enforce AppArmor: %v", err),
 	}
 }
+
+func isHTTPResponseError(err error) bool {
+	if err == nil {
+		return false
+	}
+	urlErr := &url.Error{}
+	if !errors.As(err, &urlErr) {
+		return false
+	}
+	return strings.Contains(urlErr.Err.Error(), "server gave HTTP response to HTTPS client")
+}

pkg/kubelet/lifecycle/handlers_test.go

@@ -19,7 +19,9 @@ package lifecycle
 import (
 	"fmt"
 	"io"
+	"net"
 	"net/http"
+	"net/http/httptest"
 	"reflect"
 	"strings"
 	"testing"
@@ -745,3 +747,72 @@ func TestRunHandlerHttpFailure(t *testing.T) {
 		t.Errorf("unexpected url: %s", fakeHTTPGetter.url)
 	}
 }
+
+func TestRunHandlerHttpsFailureFallback(t *testing.T) {
+	var actualHeaders http.Header
+	srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		actualHeaders = r.Header.Clone()
+	}))
+	defer srv.Close()
+	_, port, err := net.SplitHostPort(srv.Listener.Addr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fakePodStatusProvider := stubPodStatusProvider("127.0.0.1")
+
+	handlerRunner := NewHandlerRunner(srv.Client(), &fakeContainerCommandRunner{}, fakePodStatusProvider).(*handlerRunner)
+
+	containerName := "containerFoo"
+	containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"}
+	container := v1.Container{
+		Name: containerName,
+		Lifecycle: &v1.Lifecycle{
+			PostStart: &v1.LifecycleHandler{
+				HTTPGet: &v1.HTTPGetAction{
+					// set the scheme to https to ensure it falls back to HTTP.
+					Scheme: "https",
+					Host:   "127.0.0.1",
+					Port:   intstr.FromString(port),
+					Path:   "bar",
+					HTTPHeaders: []v1.HTTPHeader{
+						{
+							Name:  "Authorization",
+							Value: "secret",
+						},
+					},
+				},
+			},
+		},
+	}
+	pod := v1.Pod{}
+	pod.ObjectMeta.Name = "podFoo"
+	pod.ObjectMeta.Namespace = "nsFoo"
+	pod.Spec.Containers = []v1.Container{container}
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)()
+	msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart)
+
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if msg != "" {
+		t.Errorf("unexpected error message: %q", msg)
+	}
+	if actualHeaders.Get("Authorization") != "" {
+		t.Error("unexpected Authorization header")
+	}
+}
+
+func TestIsHTTPResponseError(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {}))
+	defer s.Close()
+	req, err := http.NewRequest("GET", s.URL, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.URL.Scheme = "https"
+	_, err = http.DefaultClient.Do(req)
+	if !isHTTPResponseError(err) {
+		t.Errorf("unexpected http response error: %v", err)
+	}
+}