From eb967ad1433250b39af2722a52d4568911c29493 Mon Sep 17 00:00:00 2001 From: Jan Chaloupka Date: Tue, 26 Jul 2016 12:21:17 +0200 Subject: [PATCH] kubelet: introduce --protect-kernel-defaults to make the KernelTunableBehavior configurable --- cmd/kubelet/app/options/options.go | 1 + cmd/kubelet/app/server.go | 23 +++++++++++++--------- hack/verify-flags/known-flags.txt | 1 + pkg/apis/componentconfig/types.go | 2 ++ pkg/apis/componentconfig/v1alpha1/types.go | 2 ++ pkg/kubelet/cm/container_manager.go | 13 ++++++------ pkg/kubelet/cm/container_manager_linux.go | 8 +++++--- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 35cb99cf8c7..af08bd77d43 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -181,4 +181,5 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.Int32Var(&s.EvictionMaxPodGracePeriod, "eviction-max-pod-grace-period", s.EvictionMaxPodGracePeriod, "Maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met. If negative, defer to pod specified value.") fs.StringVar(&s.EvictionMinimumReclaim, "eviction-minimum-reclaim", s.EvictionMinimumReclaim, "A set of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.") fs.Int32Var(&s.PodsPerCore, "pods-per-core", s.PodsPerCore, "Number of Pods per core that can run on this Kubelet. The total number of Pods on this Kubelet cannot exceed max-pods, so max-pods will be used if this calculation results in a larger number of Pods allowed on the Kubelet. A value of 0 disables this limit.") + fs.BoolVar(&s.ProtectKernelDefaults, "protect-kernel-defaults", s.ProtectKernelDefaults, "Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults.") } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index e8ab609c348..438c8b5f991 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -280,9 +280,10 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { HairpinMode: s.HairpinMode, BabysitDaemons: s.BabysitDaemons, ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay, - NodeIP: net.ParseIP(s.NodeIP), - EvictionConfig: evictionConfig, - PodsPerCore: int(s.PodsPerCore), + NodeIP: net.ParseIP(s.NodeIP), + EvictionConfig: evictionConfig, + PodsPerCore: int(s.PodsPerCore), + ProtectKernelDefaults: s.ProtectKernelDefaults, }, nil } @@ -370,12 +371,13 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) { return fmt.Errorf("invalid configuration: system container was specified and cgroup root was not specified") } kcfg.ContainerManager, err = cm.NewContainerManager(kcfg.Mounter, kcfg.CAdvisorInterface, cm.NodeConfig{ - RuntimeCgroupsName: kcfg.RuntimeCgroups, - SystemCgroupsName: kcfg.SystemCgroups, - KubeletCgroupsName: kcfg.KubeletCgroups, - ContainerRuntime: kcfg.ContainerRuntime, - CgroupsPerQOS: kcfg.CgroupsPerQOS, - CgroupRoot: kcfg.CgroupRoot, + RuntimeCgroupsName: kcfg.RuntimeCgroups, + SystemCgroupsName: kcfg.SystemCgroups, + KubeletCgroupsName: kcfg.KubeletCgroups, + ContainerRuntime: kcfg.ContainerRuntime, + CgroupsPerQOS: kcfg.CgroupsPerQOS, + CgroupRoot: kcfg.CgroupRoot, + ProtectKernelDefaults: kcfg.ProtectKernelDefaults, }) if err != nil { return err @@ -624,6 +626,7 @@ func SimpleKubelet(client *clientset.Clientset, OutOfDiskTransitionFrequency: outOfDiskTransitionFrequency, EvictionConfig: evictionConfig, PodsPerCore: podsPerCore, + ProtectKernelDefaults: false, } return &kcfg } @@ -876,6 +879,8 @@ type KubeletConfig struct { HairpinMode string BabysitDaemons bool Options []kubelet.Option + + ProtectKernelDefaults bool } func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) { diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 6e26d2a16b8..7b973024d5e 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -369,6 +369,7 @@ portal-net prepull-images private-mountns prom-push-gateway +protect-kernel-defaults proto-import proxy-bindall proxy-kubeconfig diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 5b6fce71e1b..68de7308c37 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -396,6 +396,8 @@ type KubeletConfiguration struct { // Currently only cpu and memory are supported. [default=none] // See http://releases.k8s.io/HEAD/docs/user-guide/compute-resources.md for more detail. KubeReserved utilconfig.ConfigurationMap `json:"kubeReserved"` + // Default behaviour for kernel tuning + ProtectKernelDefaults bool `json:"protectKernelDefaults"` } type KubeSchedulerConfiguration struct { diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 681bce3d67b..05945cc2c2f 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -451,4 +451,6 @@ type KubeletConfiguration struct { // Currently only cpu and memory are supported. [default=none] // See http://releases.k8s.io/HEAD/docs/user-guide/compute-resources.md for more detail. KubeReserved map[string]string `json:"kubeReserved"` + // Default behaviour for kernel tuning + ProtectKernelDefaults bool `json:"protectKernelDefaults"` } diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 58ed5362026..408636d24fb 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -39,12 +39,13 @@ type ContainerManager interface { } type NodeConfig struct { - RuntimeCgroupsName string - SystemCgroupsName string - KubeletCgroupsName string - ContainerRuntime string - CgroupsPerQOS bool - CgroupRoot string + RuntimeCgroupsName string + SystemCgroupsName string + KubeletCgroupsName string + ContainerRuntime string + CgroupsPerQOS bool + CgroupRoot string + ProtectKernelDefaults bool } type Status struct { diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 2230db98d69..a4f16b715d1 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -199,7 +199,6 @@ func createManager(containerName string) *fs.Manager { } } -// TODO: plumb this up as a flag to Kubelet in a future PR type KernelTunableBehavior string const ( @@ -288,8 +287,11 @@ func (cm *containerManagerImpl) setupNode() error { if !f.cpuHardcapping { cm.status.SoftRequirements = fmt.Errorf("CPU hardcapping unsupported") } - // TODO: plumb kernel tunable options into container manager, right now, we modify by default - if err := setupKernelTunables(KernelTunableModify); err != nil { + b := KernelTunableModify + if cm.GetNodeConfig().ProtectKernelDefaults { + b = KernelTunableError + } + if err := setupKernelTunables(b); err != nil { return err }