github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 12:43:23 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #108107 from aojea/kubelet_http2_heartbeat

Browse Source 
kubelet apiserver: be gentle closing connections on heartbeat failures
This commit is contained in:
Kubernetes Prow Robot 2022-03-09 01:32:12 -08:00 committed by GitHub
commit a41f9e976d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cmd/kubelet/app/server.go
View File

@ -557,14 +557,14 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
klog.InfoS("Standalone mode, no API client")
case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil:
clientConfig, closeAllConns, err := buildKubeletClientConfig(ctx, s, nodeName)
clientConfig, onHeartbeatFailure, err := buildKubeletClientConfig(ctx, s, nodeName)
if err != nil {
return err
}
if closeAllConns == nil {
return errors.New("closeAllConns must be a valid function other than nil")
if onHeartbeatFailure == nil {
return errors.New("onHeartbeatFailure must be a valid function other than nil")
}
kubeDeps.OnHeartbeatFailure = closeAllConns
kubeDeps.OnHeartbeatFailure = onHeartbeatFailure
kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig)
if err != nil {
@ -848,11 +848,24 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod
if err != nil {
return nil, nil, err
}
var onHeartbeatFailure func()
// Kubelet needs to be able to recover from stale http connections.
// HTTP2 has a mechanism to detect broken connections by sending periodical pings.
// HTTP1 only can have one persistent connection, and it will close all Idle connections
// once the Kubelet heartbeat fails. However, since there are many edge cases that we can't
// control, users can still opt-in to the previous behavior for closing the connections by
// setting the environment variable DISABLE_HTTP2.
if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
klog.InfoS("HTTP2 has been explicitly disabled, Kubelet will forcefully close active connections on heartbeat failures")
onHeartbeatFailure = closeAllConns
} else {
onHeartbeatFailure = func() { utilnet.CloseIdleConnectionsFor(transportConfig.Transport) }
}
klog.V(2).InfoS("Starting client certificate rotation")
clientCertificateManager.Start()
return transportConfig, closeAllConns, nil
return transportConfig, onHeartbeatFailure, nil
}
if len(s.BootstrapKubeconfig) > 0 {
@ -876,19 +889,19 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod
// once the Kubelet heartbeat fails. However, since there are many edge cases that we can't
// control, users can still opt-in to the previous behavior for closing the connections by
// setting the environment variable DISABLE_HTTP2.
var closeAllConns func()
var onHeartbeatFailure func()
if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
klog.InfoS("HTTP2 has been explicitly disabled, updating Kubelet client Dialer to forcefully close active connections on heartbeat failures")
closeAllConns, err = updateDialer(clientConfig)
onHeartbeatFailure, err = updateDialer(clientConfig)
if err != nil {
return nil, nil, err
}
} else {
closeAllConns = func() {
onHeartbeatFailure = func() {
utilnet.CloseIdleConnectionsFor(clientConfig.Transport)
}
}
return clientConfig, closeAllConns, nil
return clientConfig, onHeartbeatFailure, nil
}
// updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections.