mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 12:43:23 +00:00
Merge pull request #108107 from aojea/kubelet_http2_heartbeat
kubelet apiserver: be gentle closing connections on heartbeat failures
This commit is contained in:
commit
a41f9e976d
@ -557,14 +557,14 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
|
||||
klog.InfoS("Standalone mode, no API client")
|
||||
|
||||
case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil:
|
||||
clientConfig, closeAllConns, err := buildKubeletClientConfig(ctx, s, nodeName)
|
||||
clientConfig, onHeartbeatFailure, err := buildKubeletClientConfig(ctx, s, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeAllConns == nil {
|
||||
return errors.New("closeAllConns must be a valid function other than nil")
|
||||
if onHeartbeatFailure == nil {
|
||||
return errors.New("onHeartbeatFailure must be a valid function other than nil")
|
||||
}
|
||||
kubeDeps.OnHeartbeatFailure = closeAllConns
|
||||
kubeDeps.OnHeartbeatFailure = onHeartbeatFailure
|
||||
|
||||
kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig)
|
||||
if err != nil {
|
||||
@ -848,11 +848,24 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var onHeartbeatFailure func()
|
||||
// Kubelet needs to be able to recover from stale http connections.
|
||||
// HTTP2 has a mechanism to detect broken connections by sending periodical pings.
|
||||
// HTTP1 only can have one persistent connection, and it will close all Idle connections
|
||||
// once the Kubelet heartbeat fails. However, since there are many edge cases that we can't
|
||||
// control, users can still opt-in to the previous behavior for closing the connections by
|
||||
// setting the environment variable DISABLE_HTTP2.
|
||||
if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
|
||||
klog.InfoS("HTTP2 has been explicitly disabled, Kubelet will forcefully close active connections on heartbeat failures")
|
||||
onHeartbeatFailure = closeAllConns
|
||||
} else {
|
||||
onHeartbeatFailure = func() { utilnet.CloseIdleConnectionsFor(transportConfig.Transport) }
|
||||
}
|
||||
|
||||
klog.V(2).InfoS("Starting client certificate rotation")
|
||||
clientCertificateManager.Start()
|
||||
|
||||
return transportConfig, closeAllConns, nil
|
||||
return transportConfig, onHeartbeatFailure, nil
|
||||
}
|
||||
|
||||
if len(s.BootstrapKubeconfig) > 0 {
|
||||
@ -876,19 +889,19 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod
|
||||
// once the Kubelet heartbeat fails. However, since there are many edge cases that we can't
|
||||
// control, users can still opt-in to the previous behavior for closing the connections by
|
||||
// setting the environment variable DISABLE_HTTP2.
|
||||
var closeAllConns func()
|
||||
var onHeartbeatFailure func()
|
||||
if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
|
||||
klog.InfoS("HTTP2 has been explicitly disabled, updating Kubelet client Dialer to forcefully close active connections on heartbeat failures")
|
||||
closeAllConns, err = updateDialer(clientConfig)
|
||||
onHeartbeatFailure, err = updateDialer(clientConfig)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
} else {
|
||||
closeAllConns = func() {
|
||||
onHeartbeatFailure = func() {
|
||||
utilnet.CloseIdleConnectionsFor(clientConfig.Transport)
|
||||
}
|
||||
}
|
||||
return clientConfig, closeAllConns, nil
|
||||
return clientConfig, onHeartbeatFailure, nil
|
||||
}
|
||||
|
||||
// updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections.
|
||||
|
Loading…
Reference in New Issue
Block a user