Merge pull request #99471 from neolit123/1.21-default-systemd-cgroup-driver

kubeadm: set the kubelet cgroup driver to "systemd" during "init"

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -451,6 +451,12 @@ type ComponentConfig interface {
 
 	// SetUserSupplied sets the state of the component config "user supplied" flag to, either true, or false.
 	SetUserSupplied(userSupplied bool)
+
+	// Set can be used to set the internal configuration in the ComponentConfig
+	Set(interface{})
+
+	// Get can be used to get the internal configuration in the ComponentConfig
+	Get() interface{}
 }
 
 // ComponentConfigMap is a map between a group name (as in GVK group) and a ComponentConfig

cmd/kubeadm/app/cmd/init.go

@@ -37,6 +37,7 @@ import (
 	phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/init"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
@@ -336,6 +337,12 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 		return nil, err
 	}
 
+	// For new clusters we want to set the kubelet cgroup driver to "systemd" unless the user is explicit about it.
+	// Currently this cannot be as part of the kubelet defaulting (Default()) because the function is called for
+	// upgrades too, which can break existing nodes after a kubelet restart.
+	// TODO: https://github.com/kubernetes/kubeadm/issues/2376
+	componentconfigs.MutateCgroupDriver(&cfg.ClusterConfiguration)
+
 	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors)
 	if err != nil {
 		return nil, err

cmd/kubeadm/app/cmd/upgrade/plan.go

@@ -91,6 +91,10 @@ func runPlan(flags *planFlags, args []string) error {
 		return errors.WithMessage(err, "[upgrade/versions] FATAL")
 	}
 
+	// Warn if the kubelet component config is missing an explicit 'cgroupDriver'.
+	// TODO: https://github.com/kubernetes/kubeadm/issues/2376
+	componentconfigs.WarnCgroupDriver(&cfg.ClusterConfiguration)
+
 	// No upgrades available
 	if len(availUpgrades) == 0 {
 		klog.V(1).Infoln("[upgrade/plan] Awesome, you're up-to-date! Enjoy!")

cmd/kubeadm/app/componentconfigs/fakeconfig_test.go

@@ -96,6 +96,14 @@ func (cc *clusterConfig) Unmarshal(docmap kubeadmapi.DocumentMap) error {
 	return cc.configBase.Unmarshal(docmap, &cc.config)
 }
 
+func (cc *clusterConfig) Get() interface{} {
+	return &cc.config
+}
+
+func (cc *clusterConfig) Set(cfg interface{}) {
+	cc.config = *cfg.(*kubeadmapiv1.ClusterConfiguration)
+}
+
 func (cc *clusterConfig) Default(_ *kubeadmapi.ClusterConfiguration, _ *kubeadmapi.APIEndpoint, _ *kubeadmapi.NodeRegistrationOptions) {
 	cc.config.ClusterName = "foo"
 	cc.config.KubernetesVersion = "bar"

cmd/kubeadm/app/componentconfigs/kubelet.go

@@ -17,8 +17,10 @@ limitations under the License.
 package componentconfigs
 
 import (
+	"fmt"
 	"path/filepath"
 
+	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/version"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
@@ -104,6 +106,14 @@ func (kc *kubeletConfig) Unmarshal(docmap kubeadmapi.DocumentMap) error {
 	return kc.configBase.Unmarshal(docmap, &kc.config)
 }
 
+func (kc *kubeletConfig) Get() interface{} {
+	return &kc.config
+}
+
+func (kc *kubeletConfig) Set(cfg interface{}) {
+	kc.config = *cfg.(*kubeletconfig.KubeletConfiguration)
+}
+
 func (kc *kubeletConfig) Default(cfg *kubeadmapi.ClusterConfiguration, _ *kubeadmapi.APIEndpoint, nodeRegOpts *kubeadmapi.NodeRegistrationOptions) {
 	const kind = "KubeletConfiguration"
 
@@ -231,3 +241,54 @@ func isServiceActive(name string) (bool, error) {
 	}
 	return initSystem.ServiceIsActive(name), nil
 }
+
+// TODO: https://github.com/kubernetes/kubeadm/issues/2376
+const cgroupDriverSystemd = "systemd"
+
+// MutateCgroupDriver can be called to set the KubeletConfiguration cgroup driver to systemd.
+// Currently this cannot be as part of Default() because the function is called for
+// upgrades too, which can break existing nodes after a kubelet restart.
+// TODO: https://github.com/kubernetes/kubeadm/issues/2376
+func MutateCgroupDriver(cfg *kubeadmapi.ClusterConfiguration) {
+	cc, k, err := getKubeletConfig(cfg)
+	if err != nil {
+		klog.Warningf(err.Error())
+		return
+	}
+	if len(k.CgroupDriver) == 0 {
+		klog.V(1).Infof("setting the KubeletConfiguration cgroupDriver to %q", cgroupDriverSystemd)
+		k.CgroupDriver = cgroupDriverSystemd
+		cc.Set(k)
+	}
+}
+
+// WarnCgroupDriver prints a warning in case the user is not explicit
+// about the cgroupDriver value in the KubeletConfiguration.
+// TODO: https://github.com/kubernetes/kubeadm/issues/2376
+func WarnCgroupDriver(cfg *kubeadmapi.ClusterConfiguration) {
+	_, k, err := getKubeletConfig(cfg)
+	if err != nil {
+		klog.Warningf(err.Error())
+		return
+	}
+	if len(k.CgroupDriver) == 0 {
+		klog.Warningf("The 'cgroupDriver' value in the KubeletConfiguration is empty. " +
+			"Starting from 1.22, 'kubeadm upgrade' will default an empty value to the 'systemd' cgroup driver. " +
+			"The cgroup driver between the container runtime and the kubelet must match! " +
+			"To learn more about this see: https://kubernetes.io/docs/setup/production-environment/container-runtimes/")
+	}
+}
+
+// TODO: https://github.com/kubernetes/kubeadm/issues/2376
+func getKubeletConfig(cfg *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, *kubeletconfig.KubeletConfiguration, error) {
+	errStr := fmt.Sprintf("setting the KubeletConfiguration cgroupDriver to %q failed unexpectedly", cgroupDriverSystemd)
+	cc, ok := cfg.ComponentConfigs[KubeletGroup]
+	if !ok {
+		return nil, nil, errors.Errorf("%s: %s", errStr, "missing kubelet component config")
+	}
+	k, ok := cc.Get().(*kubeletconfig.KubeletConfiguration)
+	if !ok {
+		return nil, nil, errors.Errorf("%s: %s", errStr, "incompatible KubeletConfiguration")
+	}
+	return cc, k, nil
+}

cmd/kubeadm/app/componentconfigs/kubeproxy.go

@@ -83,6 +83,14 @@ func kubeProxyDefaultBindAddress(localAdvertiseAddress string) string {
 	return kubeadmapiv1beta2.DefaultProxyBindAddressv6
 }
 
+func (kp *kubeProxyConfig) Get() interface{} {
+	return &kp.config
+}
+
+func (kp *kubeProxyConfig) Set(cfg interface{}) {
+	kp.config = *cfg.(*kubeproxyconfig.KubeProxyConfiguration)
+}
+
 func (kp *kubeProxyConfig) Default(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint, _ *kubeadmapi.NodeRegistrationOptions) {
 	const kind = "KubeProxyConfiguration"