github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 13:37:30 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #108107 from aojea/kubelet_http2_heartbeat

Browse Source 
kubelet apiserver: be gentle closing connections on heartbeat failures
This commit is contained in:
Kubernetes Prow Robot 2022-03-09 01:32:12 -08:00 committed by GitHub
commit a41f9e976d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cmd/kubelet/app/server.go
View File

@ -557,14 +557,14 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
klog.InfoS("Standalone mode, no API client") klog.InfoS("Standalone mode, no API client")
case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil: case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil:
clientConfig, closeAllConns, err := buildKubeletClientConfig(ctx, s, nodeName) clientConfig, onHeartbeatFailure, err := buildKubeletClientConfig(ctx, s, nodeName)
if err != nil { if err != nil {
return err return err
} }
if closeAllConns == nil { if onHeartbeatFailure == nil {
return errors.New("closeAllConns must be a valid function other than nil") return errors.New("onHeartbeatFailure must be a valid function other than nil")
} }
kubeDeps.OnHeartbeatFailure = closeAllConns kubeDeps.OnHeartbeatFailure = onHeartbeatFailure
kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig) kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig)
if err != nil { if err != nil {
@ -848,11 +848,24 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
var onHeartbeatFailure func()
// Kubelet needs to be able to recover from stale http connections.
// HTTP2 has a mechanism to detect broken connections by sending periodical pings.
// HTTP1 only can have one persistent connection, and it will close all Idle connections
// once the Kubelet heartbeat fails. However, since there are many edge cases that we can't
// control, users can still opt-in to the previous behavior for closing the connections by
// setting the environment variable DISABLE_HTTP2.
if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
klog.InfoS("HTTP2 has been explicitly disabled, Kubelet will forcefully close active connections on heartbeat failures")
onHeartbeatFailure = closeAllConns
} else {
onHeartbeatFailure = func() { utilnet.CloseIdleConnectionsFor(transportConfig.Transport) }
}
klog.V(2).InfoS("Starting client certificate rotation") klog.V(2).InfoS("Starting client certificate rotation")
clientCertificateManager.Start() clientCertificateManager.Start()
return transportConfig, closeAllConns, nil return transportConfig, onHeartbeatFailure, nil
} }
if len(s.BootstrapKubeconfig) > 0 { if len(s.BootstrapKubeconfig) > 0 {
@ -876,19 +889,19 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod
// once the Kubelet heartbeat fails. However, since there are many edge cases that we can't // once the Kubelet heartbeat fails. However, since there are many edge cases that we can't
// control, users can still opt-in to the previous behavior for closing the connections by // control, users can still opt-in to the previous behavior for closing the connections by
// setting the environment variable DISABLE_HTTP2. // setting the environment variable DISABLE_HTTP2.
var closeAllConns func() var onHeartbeatFailure func()
if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 { if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
klog.InfoS("HTTP2 has been explicitly disabled, updating Kubelet client Dialer to forcefully close active connections on heartbeat failures") klog.InfoS("HTTP2 has been explicitly disabled, updating Kubelet client Dialer to forcefully close active connections on heartbeat failures")
closeAllConns, err = updateDialer(clientConfig) onHeartbeatFailure, err = updateDialer(clientConfig)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
} else { } else {
closeAllConns = func() { onHeartbeatFailure = func() {
utilnet.CloseIdleConnectionsFor(clientConfig.Transport) utilnet.CloseIdleConnectionsFor(clientConfig.Transport)
} }
} }
return clientConfig, closeAllConns, nil return clientConfig, onHeartbeatFailure, nil
} }
// updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections. // updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections.