diff --git a/pkg/kubelet/lifecycle/handlers.go b/pkg/kubelet/lifecycle/handlers.go index 8ec4d19f7cd..a8a98e012a6 100644 --- a/pkg/kubelet/lifecycle/handlers.go +++ b/pkg/kubelet/lifecycle/handlers.go @@ -17,11 +17,15 @@ limitations under the License. package lifecycle import ( + "context" + "errors" "fmt" "io" "net" "net/http" + "net/url" "strconv" + "strings" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -132,6 +136,23 @@ func (hr *handlerRunner) runHTTPHandler(pod *v1.Pod, container *v1.Container, ha } resp, err := hr.httpDoer.Do(req) discardHTTPRespBody(resp) + + if isHTTPResponseError(err) { + // TODO: emit an event about the fallback + // TODO: increment a metric about the fallback + klog.V(1).ErrorS(err, "HTTPS request to lifecycle hook got HTTP response, retrying with HTTP.", "pod", klog.KObj(pod), "host", req.URL.Host) + + req := req.Clone(context.Background()) + req.URL.Scheme = "http" + req.Header.Del("Authorization") + resp, httpErr := hr.httpDoer.Do(req) + + // clear err since the fallback succeeded + if httpErr == nil { + err = nil + } + discardHTTPRespBody(resp) + } return err } @@ -201,3 +222,14 @@ func (a *appArmorAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult { Message: fmt.Sprintf("Cannot enforce AppArmor: %v", err), } } + +func isHTTPResponseError(err error) bool { + if err == nil { + return false + } + urlErr := &url.Error{} + if !errors.As(err, &urlErr) { + return false + } + return strings.Contains(urlErr.Err.Error(), "server gave HTTP response to HTTPS client") +} diff --git a/pkg/kubelet/lifecycle/handlers_test.go b/pkg/kubelet/lifecycle/handlers_test.go index 3989a421b55..0dbdb24941e 100644 --- a/pkg/kubelet/lifecycle/handlers_test.go +++ b/pkg/kubelet/lifecycle/handlers_test.go @@ -19,7 +19,9 @@ package lifecycle import ( "fmt" "io" + "net" "net/http" + "net/http/httptest" "reflect" "strings" "testing" @@ -745,3 +747,72 @@ func TestRunHandlerHttpFailure(t *testing.T) { t.Errorf("unexpected url: %s", fakeHTTPGetter.url) } } + +func TestRunHandlerHttpsFailureFallback(t *testing.T) { + var actualHeaders http.Header + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + actualHeaders = r.Header.Clone() + })) + defer srv.Close() + _, port, err := net.SplitHostPort(srv.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + + fakePodStatusProvider := stubPodStatusProvider("127.0.0.1") + + handlerRunner := NewHandlerRunner(srv.Client(), &fakeContainerCommandRunner{}, fakePodStatusProvider).(*handlerRunner) + + containerName := "containerFoo" + containerID := kubecontainer.ContainerID{Type: "test", ID: "abc1234"} + container := v1.Container{ + Name: containerName, + Lifecycle: &v1.Lifecycle{ + PostStart: &v1.LifecycleHandler{ + HTTPGet: &v1.HTTPGetAction{ + // set the scheme to https to ensure it falls back to HTTP. + Scheme: "https", + Host: "127.0.0.1", + Port: intstr.FromString(port), + Path: "bar", + HTTPHeaders: []v1.HTTPHeader{ + { + Name: "Authorization", + Value: "secret", + }, + }, + }, + }, + }, + } + pod := v1.Pod{} + pod.ObjectMeta.Name = "podFoo" + pod.ObjectMeta.Namespace = "nsFoo" + pod.Spec.Containers = []v1.Container{container} + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, true)() + msg, err := handlerRunner.Run(containerID, &pod, &container, container.Lifecycle.PostStart) + + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if msg != "" { + t.Errorf("unexpected error message: %q", msg) + } + if actualHeaders.Get("Authorization") != "" { + t.Error("unexpected Authorization header") + } +} + +func TestIsHTTPResponseError(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + defer s.Close() + req, err := http.NewRequest("GET", s.URL, nil) + if err != nil { + t.Fatal(err) + } + req.URL.Scheme = "https" + _, err = http.DefaultClient.Do(req) + if !isHTTPResponseError(err) { + t.Errorf("unexpected http response error: %v", err) + } +}