Merge pull request #27874 from ingvagabund/kubelet-kernel-tunning-behaviour-new-flags

Automatic merge from submit-queue

[kubelet] Introduce --protect-kernel-defaults flag to make the tunable behaviour configurable

Let's make the default behaviour of kernel tuning configurable. The default behaviour is kept modify as has been so far.

cmd/kubelet/app/options/options.go

@@ -183,4 +183,5 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&s.EvictionMaxPodGracePeriod, "eviction-max-pod-grace-period", s.EvictionMaxPodGracePeriod, "Maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met.  If negative, defer to pod specified value.")
 	fs.StringVar(&s.EvictionMinimumReclaim, "eviction-minimum-reclaim", s.EvictionMinimumReclaim, "A set of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.")
 	fs.Int32Var(&s.PodsPerCore, "pods-per-core", s.PodsPerCore, "Number of Pods per core that can run on this Kubelet. The total number of Pods on this Kubelet cannot exceed max-pods, so max-pods will be used if this calculation results in a larger number of Pods allowed on the Kubelet. A value of 0 disables this limit.")
+	fs.BoolVar(&s.ProtectKernelDefaults, "protect-kernel-defaults", s.ProtectKernelDefaults, "Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults.")
 }

cmd/kubelet/app/server.go

@@ -282,9 +282,10 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 		HairpinMode:                    s.HairpinMode,
 		BabysitDaemons:                 s.BabysitDaemons,
 		ExperimentalFlannelOverlay:     s.ExperimentalFlannelOverlay,
-		NodeIP:         net.ParseIP(s.NodeIP),
-		EvictionConfig: evictionConfig,
-		PodsPerCore:    int(s.PodsPerCore),
+		NodeIP:                net.ParseIP(s.NodeIP),
+		EvictionConfig:        evictionConfig,
+		PodsPerCore:           int(s.PodsPerCore),
+		ProtectKernelDefaults: s.ProtectKernelDefaults,
 	}, nil
 }
 
@@ -382,12 +383,13 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) {
 			return fmt.Errorf("invalid configuration: system container was specified and cgroup root was not specified")
 		}
 		kcfg.ContainerManager, err = cm.NewContainerManager(kcfg.Mounter, kcfg.CAdvisorInterface, cm.NodeConfig{
-			RuntimeCgroupsName: kcfg.RuntimeCgroups,
-			SystemCgroupsName:  kcfg.SystemCgroups,
-			KubeletCgroupsName: kcfg.KubeletCgroups,
-			ContainerRuntime:   kcfg.ContainerRuntime,
-			CgroupsPerQOS:      kcfg.CgroupsPerQOS,
-			CgroupRoot:         kcfg.CgroupRoot,
+			RuntimeCgroupsName:    kcfg.RuntimeCgroups,
+			SystemCgroupsName:     kcfg.SystemCgroups,
+			KubeletCgroupsName:    kcfg.KubeletCgroups,
+			ContainerRuntime:      kcfg.ContainerRuntime,
+			CgroupsPerQOS:         kcfg.CgroupsPerQOS,
+			CgroupRoot:            kcfg.CgroupRoot,
+			ProtectKernelDefaults: kcfg.ProtectKernelDefaults,
 		})
 		if err != nil {
 			return err
@@ -636,6 +638,7 @@ func SimpleKubelet(client *clientset.Clientset,
 		OutOfDiskTransitionFrequency: outOfDiskTransitionFrequency,
 		EvictionConfig:               evictionConfig,
 		PodsPerCore:                  podsPerCore,
+		ProtectKernelDefaults:        false,
 	}
 	return &kcfg
 }
@@ -888,6 +891,8 @@ type KubeletConfig struct {
 	HairpinMode                string
 	BabysitDaemons             bool
 	Options                    []kubelet.Option
+
+	ProtectKernelDefaults bool
 }
 
 func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) {

hack/verify-flags/known-flags.txt

@@ -374,6 +374,7 @@ portal-net
 prepull-images
 private-mountns
 prom-push-gateway
+protect-kernel-defaults
 proto-import
 proxy-bindall
 proxy-kubeconfig

pkg/apis/componentconfig/types.go

@@ -397,6 +397,8 @@ type KubeletConfiguration struct {
 	// Currently only cpu and memory are supported. [default=none]
 	// See http://releases.k8s.io/HEAD/docs/user-guide/compute-resources.md for more detail.
 	KubeReserved utilconfig.ConfigurationMap `json:"kubeReserved"`
+	// Default behaviour for kernel tuning
+	ProtectKernelDefaults bool `json:"protectKernelDefaults"`
 }
 
 type KubeSchedulerConfiguration struct {

pkg/apis/componentconfig/v1alpha1/types.go

@@ -452,4 +452,6 @@ type KubeletConfiguration struct {
 	// Currently only cpu and memory are supported. [default=none]
 	// See http://releases.k8s.io/HEAD/docs/user-guide/compute-resources.md for more detail.
 	KubeReserved map[string]string `json:"kubeReserved"`
+	// Default behaviour for kernel tuning
+	ProtectKernelDefaults bool `json:"protectKernelDefaults"`
 }

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -319,6 +319,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	} else {
 		out.KubeReserved = nil
 	}
+	out.ProtectKernelDefaults = in.ProtectKernelDefaults
 	return nil
 }
 
@@ -483,6 +484,7 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	} else {
 		out.KubeReserved = nil
 	}
+	out.ProtectKernelDefaults = in.ProtectKernelDefaults
 	return nil
 }
 

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -377,6 +377,7 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 		} else {
 			out.KubeReserved = nil
 		}
+		out.ProtectKernelDefaults = in.ProtectKernelDefaults
 		return nil
 	}
 }

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -325,6 +325,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
 		} else {
 			out.KubeReserved = nil
 		}
+		out.ProtectKernelDefaults = in.ProtectKernelDefaults
 		return nil
 	}
 }

pkg/kubelet/cm/container_manager.go

@@ -39,12 +39,13 @@ type ContainerManager interface {
 }
 
 type NodeConfig struct {
-	RuntimeCgroupsName string
-	SystemCgroupsName  string
-	KubeletCgroupsName string
-	ContainerRuntime   string
-	CgroupsPerQOS      bool
-	CgroupRoot         string
+	RuntimeCgroupsName    string
+	SystemCgroupsName     string
+	KubeletCgroupsName    string
+	ContainerRuntime      string
+	CgroupsPerQOS         bool
+	CgroupRoot            string
+	ProtectKernelDefaults bool
 }
 
 type Status struct {

pkg/kubelet/cm/container_manager_linux.go

@@ -199,7 +199,6 @@ func createManager(containerName string) *fs.Manager {
 	}
 }
 
-// TODO: plumb this up as a flag to Kubelet in a future PR
 type KernelTunableBehavior string
 
 const (
@@ -288,8 +287,11 @@ func (cm *containerManagerImpl) setupNode() error {
 	if !f.cpuHardcapping {
 		cm.status.SoftRequirements = fmt.Errorf("CPU hardcapping unsupported")
 	}
-	// TODO: plumb kernel tunable options into container manager, right now, we modify by default
-	if err := setupKernelTunables(KernelTunableModify); err != nil {
+	b := KernelTunableModify
+	if cm.GetNodeConfig().ProtectKernelDefaults {
+		b = KernelTunableError
+	}
+	if err := setupKernelTunables(b); err != nil {
 		return err
 	}