From b6ff320507a6504352c2d670a9b6f8ae114e1847 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Tue, 2 Feb 2021 22:34:19 +0200 Subject: [PATCH] kubeadm: set the kubelet cgroup driver to "systemd" during "init" The kubeadm documentation instructs users to set the container runtime driver to "systemd", since kubeadm manages a kubelet via the systemd init system. The kubelet default however is "cgroupfs". For new clusters set the driver to "systemd" unless the user is explicit about it. The same defaulting would not happen during "upgrade". --- cmd/kubeadm/app/apis/kubeadm/types.go | 6 ++ cmd/kubeadm/app/cmd/init.go | 7 +++ cmd/kubeadm/app/cmd/upgrade/plan.go | 4 ++ .../app/componentconfigs/fakeconfig_test.go | 8 +++ cmd/kubeadm/app/componentconfigs/kubelet.go | 61 +++++++++++++++++++ cmd/kubeadm/app/componentconfigs/kubeproxy.go | 8 +++ 6 files changed, 94 insertions(+) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 55dc999e12d..2f2b09cedac 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -451,6 +451,12 @@ type ComponentConfig interface { // SetUserSupplied sets the state of the component config "user supplied" flag to, either true, or false. SetUserSupplied(userSupplied bool) + + // Set can be used to set the internal configuration in the ComponentConfig + Set(interface{}) + + // Get can be used to get the internal configuration in the ComponentConfig + Get() interface{} } // ComponentConfigMap is a map between a group name (as in GVK group) and a ComponentConfig diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index c2a0457b6d3..7f5ecc04666 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -37,6 +37,7 @@ import ( phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/init" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" @@ -336,6 +337,12 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io return nil, err } + // For new clusters we want to set the kubelet cgroup driver to "systemd" unless the user is explicit about it. + // Currently this cannot be as part of the kubelet defaulting (Default()) because the function is called for + // upgrades too, which can break existing nodes after a kubelet restart. + // TODO: https://github.com/kubernetes/kubeadm/issues/2376 + componentconfigs.MutateCgroupDriver(&cfg.ClusterConfiguration) + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) if err != nil { return nil, err diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index 97d78b455f5..1aa0d6a304e 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -91,6 +91,10 @@ func runPlan(flags *planFlags, args []string) error { return errors.WithMessage(err, "[upgrade/versions] FATAL") } + // Warn if the kubelet component config is missing an explicit 'cgroupDriver'. + // TODO: https://github.com/kubernetes/kubeadm/issues/2376 + componentconfigs.WarnCgroupDriver(&cfg.ClusterConfiguration) + // No upgrades available if len(availUpgrades) == 0 { klog.V(1).Infoln("[upgrade/plan] Awesome, you're up-to-date! Enjoy!") diff --git a/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go b/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go index 4db2625f3ec..b070ff3d77a 100644 --- a/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go +++ b/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go @@ -96,6 +96,14 @@ func (cc *clusterConfig) Unmarshal(docmap kubeadmapi.DocumentMap) error { return cc.configBase.Unmarshal(docmap, &cc.config) } +func (cc *clusterConfig) Get() interface{} { + return &cc.config +} + +func (cc *clusterConfig) Set(cfg interface{}) { + cc.config = *cfg.(*kubeadmapiv1.ClusterConfiguration) +} + func (cc *clusterConfig) Default(_ *kubeadmapi.ClusterConfiguration, _ *kubeadmapi.APIEndpoint, _ *kubeadmapi.NodeRegistrationOptions) { cc.config.ClusterName = "foo" cc.config.KubernetesVersion = "bar" diff --git a/cmd/kubeadm/app/componentconfigs/kubelet.go b/cmd/kubeadm/app/componentconfigs/kubelet.go index ae13e8822e8..3c757fa48fc 100644 --- a/cmd/kubeadm/app/componentconfigs/kubelet.go +++ b/cmd/kubeadm/app/componentconfigs/kubelet.go @@ -17,8 +17,10 @@ limitations under the License. package componentconfigs import ( + "fmt" "path/filepath" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -104,6 +106,14 @@ func (kc *kubeletConfig) Unmarshal(docmap kubeadmapi.DocumentMap) error { return kc.configBase.Unmarshal(docmap, &kc.config) } +func (kc *kubeletConfig) Get() interface{} { + return &kc.config +} + +func (kc *kubeletConfig) Set(cfg interface{}) { + kc.config = *cfg.(*kubeletconfig.KubeletConfiguration) +} + func (kc *kubeletConfig) Default(cfg *kubeadmapi.ClusterConfiguration, _ *kubeadmapi.APIEndpoint, nodeRegOpts *kubeadmapi.NodeRegistrationOptions) { const kind = "KubeletConfiguration" @@ -231,3 +241,54 @@ func isServiceActive(name string) (bool, error) { } return initSystem.ServiceIsActive(name), nil } + +// TODO: https://github.com/kubernetes/kubeadm/issues/2376 +const cgroupDriverSystemd = "systemd" + +// MutateCgroupDriver can be called to set the KubeletConfiguration cgroup driver to systemd. +// Currently this cannot be as part of Default() because the function is called for +// upgrades too, which can break existing nodes after a kubelet restart. +// TODO: https://github.com/kubernetes/kubeadm/issues/2376 +func MutateCgroupDriver(cfg *kubeadmapi.ClusterConfiguration) { + cc, k, err := getKubeletConfig(cfg) + if err != nil { + klog.Warningf(err.Error()) + return + } + if len(k.CgroupDriver) == 0 { + klog.V(1).Infof("setting the KubeletConfiguration cgroupDriver to %q", cgroupDriverSystemd) + k.CgroupDriver = cgroupDriverSystemd + cc.Set(k) + } +} + +// WarnCgroupDriver prints a warning in case the user is not explicit +// about the cgroupDriver value in the KubeletConfiguration. +// TODO: https://github.com/kubernetes/kubeadm/issues/2376 +func WarnCgroupDriver(cfg *kubeadmapi.ClusterConfiguration) { + _, k, err := getKubeletConfig(cfg) + if err != nil { + klog.Warningf(err.Error()) + return + } + if len(k.CgroupDriver) == 0 { + klog.Warningf("The 'cgroupDriver' value in the KubeletConfiguration is empty. " + + "Starting from 1.22, 'kubeadm upgrade' will default an empty value to the 'systemd' cgroup driver. " + + "The cgroup driver between the container runtime and the kubelet must match! " + + "To learn more about this see: https://kubernetes.io/docs/setup/production-environment/container-runtimes/") + } +} + +// TODO: https://github.com/kubernetes/kubeadm/issues/2376 +func getKubeletConfig(cfg *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, *kubeletconfig.KubeletConfiguration, error) { + errStr := fmt.Sprintf("setting the KubeletConfiguration cgroupDriver to %q failed unexpectedly", cgroupDriverSystemd) + cc, ok := cfg.ComponentConfigs[KubeletGroup] + if !ok { + return nil, nil, errors.Errorf("%s: %s", errStr, "missing kubelet component config") + } + k, ok := cc.Get().(*kubeletconfig.KubeletConfiguration) + if !ok { + return nil, nil, errors.Errorf("%s: %s", errStr, "incompatible KubeletConfiguration") + } + return cc, k, nil +} diff --git a/cmd/kubeadm/app/componentconfigs/kubeproxy.go b/cmd/kubeadm/app/componentconfigs/kubeproxy.go index c53beccdce5..b423dd5a980 100644 --- a/cmd/kubeadm/app/componentconfigs/kubeproxy.go +++ b/cmd/kubeadm/app/componentconfigs/kubeproxy.go @@ -83,6 +83,14 @@ func kubeProxyDefaultBindAddress(localAdvertiseAddress string) string { return kubeadmapiv1beta2.DefaultProxyBindAddressv6 } +func (kp *kubeProxyConfig) Get() interface{} { + return &kp.config +} + +func (kp *kubeProxyConfig) Set(cfg interface{}) { + kp.config = *cfg.(*kubeproxyconfig.KubeProxyConfiguration) +} + func (kp *kubeProxyConfig) Default(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint, _ *kubeadmapi.NodeRegistrationOptions) { const kind = "KubeProxyConfiguration"