From 1162cfd3b10b3b18aaa8b3b476beea9b1738bdb4 Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Mon, 11 Jan 2016 12:28:09 -0800 Subject: [PATCH] kubelet: move most of kubelet server configuration to a config object. This is part of migrating kubelet configuration to the componentconfig api group and is preliminary to retrofitting client configuration and implementing full fledged API group mechinary. Signed-off-by: Mike Danese --- cmd/kubelet/app/options/options.go | 239 +++++++----------- cmd/kubelet/app/server.go | 22 +- contrib/mesos/pkg/executor/service/service.go | 4 +- contrib/mesos/pkg/minion/server.go | 2 +- contrib/mesos/pkg/node/statusupdater.go | 2 +- contrib/mesos/pkg/node/statusupdater_test.go | 6 +- docs/admin/kubelet.md | 11 +- pkg/apis/componentconfig/types.go | 213 ++++++++++++++++ 8 files changed, 324 insertions(+), 175 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 46e84cc94a4..ef985ae8790 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -18,11 +18,12 @@ limitations under the License. package options import ( - "net" _ "net/http/pprof" "time" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/kubelet/qos" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/master/ports" @@ -39,171 +40,101 @@ const ( // KubeletServer encapsulates all of the parameters necessary for starting up // a kubelet. These can either be set via command line or directly. type KubeletServer struct { - Address net.IP - AllowPrivileged bool - APIServerList []string - AuthPath util.StringFlag // Deprecated -- use KubeConfig instead - CAdvisorPort uint - CertDirectory string - CgroupRoot string - CloudConfigFile string - CloudProvider string - ClusterDNS net.IP - ClusterDomain string - Config string - ConfigureCBR0 bool - ContainerRuntime string - CPUCFSQuota bool - DockerDaemonContainer string - DockerEndpoint string - DockerExecHandlerName string - EnableDebuggingHandlers bool - EnableServer bool - EventBurst int - EventRecordQPS float32 - FileCheckFrequency time.Duration - HealthzBindAddress net.IP - HealthzPort int - HostnameOverride string - HostNetworkSources string - HostPIDSources string - HostIPCSources string - HTTPCheckFrequency time.Duration - ImageGCHighThresholdPercent int - ImageGCLowThresholdPercent int - KubeConfig util.StringFlag - LowDiskSpaceThresholdMB int - ManifestURL string - ManifestURLHeader string - MasterServiceNamespace string - MaxContainerCount int - MaxOpenFiles uint64 - MaxPerPodContainerCount int - MaxPods int - MinimumGCAge time.Duration - NetworkPluginDir string - NetworkPluginName string - VolumePluginDir string - NodeLabels map[string]string - NodeStatusUpdateFrequency time.Duration - OOMScoreAdj int - PodCIDR string - PodInfraContainerImage string - Port uint - ReadOnlyPort uint - RegisterNode bool - RegisterSchedulable bool - RegistryBurst int - RegistryPullQPS float64 - ResolverConfig string - ResourceContainer string - RktPath string - RktStage1Image string - RootDirectory string - RunOnce bool - StandaloneMode bool - StreamingConnectionIdleTimeout time.Duration - SyncFrequency time.Duration - SystemContainer string - TLSCertFile string - TLSPrivateKeyFile string - ReconcileCIDR bool - SystemReserved util.ConfigurationMap - KubeReserved util.ConfigurationMap + componentconfig.KubeletConfiguration + + AuthPath util.StringFlag // Deprecated -- use KubeConfig instead + KubeConfig util.StringFlag + APIServerList []string + + DockerDaemonContainer string + RunOnce bool - // Flags intended for testing - // Is the kubelet containerized? - Containerized bool // Insert a probability of random errors during calls to the master. ChaosChance float64 // Crash immediately, rather than eating panics. ReallyCrashForTesting bool - - KubeAPIQPS float32 - KubeAPIBurst int - - // Pull images one at a time. - SerializeImagePulls bool - ExperimentalFlannelOverlay bool - OutOfDiskTransitionFrequency time.Duration - NodeIP net.IP + SystemReserved util.ConfigurationMap + KubeReserved util.ConfigurationMap } // NewKubeletServer will create a new KubeletServer with default values. func NewKubeletServer() *KubeletServer { return &KubeletServer{ - Address: net.ParseIP("0.0.0.0"), - AuthPath: util.NewStringFlag("/var/lib/kubelet/kubernetes_auth"), // deprecated - CAdvisorPort: 4194, - CertDirectory: "/var/run/kubernetes", - CgroupRoot: "", - ConfigureCBR0: false, - ContainerRuntime: "docker", - CPUCFSQuota: false, - DockerDaemonContainer: "/docker-daemon", - DockerExecHandlerName: "native", - EventBurst: 10, - EventRecordQPS: 5.0, - EnableDebuggingHandlers: true, - EnableServer: true, - FileCheckFrequency: 20 * time.Second, - HealthzBindAddress: net.ParseIP("127.0.0.1"), - HealthzPort: 10248, - HostNetworkSources: kubetypes.AllSource, - HostPIDSources: kubetypes.AllSource, - HostIPCSources: kubetypes.AllSource, - HTTPCheckFrequency: 20 * time.Second, - ImageGCHighThresholdPercent: 90, - ImageGCLowThresholdPercent: 80, - KubeConfig: util.NewStringFlag("/var/lib/kubelet/kubeconfig"), - LowDiskSpaceThresholdMB: 256, - MasterServiceNamespace: api.NamespaceDefault, - MaxContainerCount: 100, - MaxPerPodContainerCount: 2, - MaxOpenFiles: 1000000, - MaxPods: 40, - MinimumGCAge: 1 * time.Minute, - NetworkPluginDir: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/", - NetworkPluginName: "", - VolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", - NodeLabels: make(map[string]string), - NodeStatusUpdateFrequency: 10 * time.Second, - OOMScoreAdj: qos.KubeletOOMScoreAdj, - PodInfraContainerImage: kubetypes.PodInfraContainerImage, - Port: ports.KubeletPort, - ReadOnlyPort: ports.KubeletReadOnlyPort, - RegisterNode: true, // will be ignored if no apiserver is configured - RegisterSchedulable: true, - RegistryBurst: 10, - RegistryPullQPS: 5.0, - ResourceContainer: "/kubelet", - RktPath: "", - RktStage1Image: "", - RootDirectory: defaultRootDir, - SerializeImagePulls: true, - StreamingConnectionIdleTimeout: 5 * time.Minute, - SyncFrequency: 1 * time.Minute, - SystemContainer: "", - ReconcileCIDR: true, - SystemReserved: make(util.ConfigurationMap), - KubeReserved: make(util.ConfigurationMap), - KubeAPIQPS: 5.0, - KubeAPIBurst: 10, - ExperimentalFlannelOverlay: experimentalFlannelOverlay, - OutOfDiskTransitionFrequency: 5 * time.Minute, + AuthPath: util.NewStringFlag("/var/lib/kubelet/kubernetes_auth"), // deprecated + KubeConfig: util.NewStringFlag("/var/lib/kubelet/kubeconfig"), + DockerDaemonContainer: "/docker-daemon", + + SystemReserved: make(util.ConfigurationMap), + KubeReserved: make(util.ConfigurationMap), + KubeletConfiguration: componentconfig.KubeletConfiguration{ + Address: "0.0.0.0", + CAdvisorPort: 4194, + CertDirectory: "/var/run/kubernetes", + CgroupRoot: "", + ConfigureCBR0: false, + ContainerRuntime: "docker", + CPUCFSQuota: false, + DockerExecHandlerName: "native", + EventBurst: 10, + EventRecordQPS: 5.0, + EnableDebuggingHandlers: true, + EnableServer: true, + FileCheckFrequency: unversioned.Duration{20 * time.Second}, + HealthzBindAddress: "127.0.0.1", + HealthzPort: 10248, + HostNetworkSources: kubetypes.AllSource, + HostPIDSources: kubetypes.AllSource, + HostIPCSources: kubetypes.AllSource, + HTTPCheckFrequency: unversioned.Duration{20 * time.Second}, + ImageGCHighThresholdPercent: 90, + ImageGCLowThresholdPercent: 80, + LowDiskSpaceThresholdMB: 256, + MasterServiceNamespace: api.NamespaceDefault, + MaxContainerCount: 100, + MaxPerPodContainerCount: 2, + MaxOpenFiles: 1000000, + MaxPods: 40, + MinimumGCAge: unversioned.Duration{1 * time.Minute}, + NetworkPluginDir: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/", + NetworkPluginName: "", + VolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", + NodeStatusUpdateFrequency: unversioned.Duration{10 * time.Second}, + NodeLabels: make(map[string]string), + OOMScoreAdj: qos.KubeletOOMScoreAdj, + PodInfraContainerImage: kubetypes.PodInfraContainerImage, + Port: ports.KubeletPort, + ReadOnlyPort: ports.KubeletReadOnlyPort, + RegisterNode: true, // will be ignored if no apiserver is configured + RegisterSchedulable: true, + RegistryBurst: 10, + RegistryPullQPS: 5.0, + ResourceContainer: "/kubelet", + RktPath: "", + RktStage1Image: "", + RootDirectory: defaultRootDir, + SerializeImagePulls: true, + StreamingConnectionIdleTimeout: unversioned.Duration{5 * time.Minute}, + SyncFrequency: unversioned.Duration{1 * time.Minute}, + SystemContainer: "", + ReconcileCIDR: true, + KubeAPIQPS: 5.0, + KubeAPIBurst: 10, + ExperimentalFlannelOverlay: experimentalFlannelOverlay, + OutOfDiskTransitionFrequency: unversioned.Duration{5 * time.Minute}, + }, } } // AddFlags adds flags for a specific KubeletServer to the specified FlagSet func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.Config, "config", s.Config, "Path to the config file or directory of files") - fs.DurationVar(&s.SyncFrequency, "sync-frequency", s.SyncFrequency, "Max period between synchronizing running containers and config") - fs.DurationVar(&s.FileCheckFrequency, "file-check-frequency", s.FileCheckFrequency, "Duration between checking config files for new data") - fs.DurationVar(&s.HTTPCheckFrequency, "http-check-frequency", s.HTTPCheckFrequency, "Duration between checking http for new data") + fs.DurationVar(&s.SyncFrequency.Duration, "sync-frequency", s.SyncFrequency.Duration, "Max period between synchronizing running containers and config") + fs.DurationVar(&s.FileCheckFrequency.Duration, "file-check-frequency", s.FileCheckFrequency.Duration, "Duration between checking config files for new data") + fs.DurationVar(&s.HTTPCheckFrequency.Duration, "http-check-frequency", s.HTTPCheckFrequency.Duration, "Duration between checking http for new data") fs.StringVar(&s.ManifestURL, "manifest-url", s.ManifestURL, "URL for accessing the container manifest") fs.StringVar(&s.ManifestURLHeader, "manifest-url-header", s.ManifestURLHeader, "HTTP header to use when accessing the manifest URL, with the key separated from the value with a ':', as in 'key:value'") fs.BoolVar(&s.EnableServer, "enable-server", s.EnableServer, "Enable the Kubelet's server") - fs.IPVar(&s.Address, "address", s.Address, "The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces)") + fs.StringVar(&s.Address, "address", s.Address, "The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces)") fs.UintVar(&s.Port, "port", s.Port, "The port for the Kubelet to serve on.") fs.UintVar(&s.ReadOnlyPort, "read-only-port", s.ReadOnlyPort, "The read-only port for the Kubelet to serve on with no authentication/authorization (set to 0 to disable)") fs.StringVar(&s.TLSCertFile, "tls-cert-file", s.TLSCertFile, ""+ @@ -227,7 +158,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&s.EventBurst, "event-burst", s.EventBurst, "Maximum size of a bursty event records, temporarily allows event records to burst to this number, while still not exceeding event-qps. Only used if --event-qps > 0") fs.BoolVar(&s.RunOnce, "runonce", s.RunOnce, "If true, exit after spawning pods from local manifests or remote urls. Exclusive with --api-servers, and --enable-server") fs.BoolVar(&s.EnableDebuggingHandlers, "enable-debugging-handlers", s.EnableDebuggingHandlers, "Enables server endpoints for log collection and local running of containers and commands") - fs.DurationVar(&s.MinimumGCAge, "minimum-container-ttl-duration", s.MinimumGCAge, "Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'") + fs.DurationVar(&s.MinimumGCAge.Duration, "minimum-container-ttl-duration", s.MinimumGCAge.Duration, "Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'") fs.IntVar(&s.MaxPerPodContainerCount, "maximum-dead-containers-per-container", s.MaxPerPodContainerCount, "Maximum number of old instances to retain per container. Each container takes up some disk space. Default: 2.") fs.IntVar(&s.MaxContainerCount, "maximum-dead-containers", s.MaxContainerCount, "Maximum number of old instances of containers to retain globally. Each container takes up some disk space. Default: 100.") fs.Var(&s.AuthPath, "auth-path", "Path to .kubernetes_auth file, specifying how to authenticate to API server.") @@ -235,17 +166,17 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.Var(&s.KubeConfig, "kubeconfig", "Path to a kubeconfig file, specifying how to authenticate to API server (the master location is set by the api-servers flag).") fs.UintVar(&s.CAdvisorPort, "cadvisor-port", s.CAdvisorPort, "The port of the localhost cAdvisor endpoint") fs.IntVar(&s.HealthzPort, "healthz-port", s.HealthzPort, "The port of the localhost healthz endpoint") - fs.IPVar(&s.HealthzBindAddress, "healthz-bind-address", s.HealthzBindAddress, "The IP address for the healthz server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)") + fs.StringVar(&s.HealthzBindAddress, "healthz-bind-address", s.HealthzBindAddress, "The IP address for the healthz server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)") fs.IntVar(&s.OOMScoreAdj, "oom-score-adj", s.OOMScoreAdj, "The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]") fs.StringSliceVar(&s.APIServerList, "api-servers", []string{}, "List of Kubernetes API servers for publishing events, and reading pods and services. (ip:port), comma separated.") fs.BoolVar(&s.RegisterNode, "register-node", s.RegisterNode, "Register the node with the apiserver (defaults to true if --api-servers is set)") fs.StringVar(&s.ClusterDomain, "cluster-domain", s.ClusterDomain, "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains") fs.StringVar(&s.MasterServiceNamespace, "master-service-namespace", s.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods") - fs.IPVar(&s.ClusterDNS, "cluster-dns", s.ClusterDNS, "IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers") - fs.DurationVar(&s.StreamingConnectionIdleTimeout, "streaming-connection-idle-timeout", s.StreamingConnectionIdleTimeout, "Maximum time a streaming connection can be idle before the connection is automatically closed. Example: '5m'") + fs.StringVar(&s.ClusterDNS, "cluster-dns", s.ClusterDNS, "IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers") + fs.DurationVar(&s.StreamingConnectionIdleTimeout.Duration, "streaming-connection-idle-timeout", s.StreamingConnectionIdleTimeout.Duration, "Maximum time a streaming connection can be idle before the connection is automatically closed. Example: '5m'") + fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s") bindableNodeLabels := util.ConfigurationMap(s.NodeLabels) fs.Var(&bindableNodeLabels, "node-labels", " Labels to add when registering the node in the cluster. Labels must are key=value pairs seperated by ','.") - fs.DurationVar(&s.NodeStatusUpdateFrequency, "node-status-update-frequency", s.NodeStatusUpdateFrequency, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s") fs.IntVar(&s.ImageGCHighThresholdPercent, "image-gc-high-threshold", s.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run. Default: 90%") fs.IntVar(&s.ImageGCLowThresholdPercent, "image-gc-low-threshold", s.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. Default: 80%") fs.IntVar(&s.LowDiskSpaceThresholdMB, "low-diskspace-threshold-mb", s.LowDiskSpaceThresholdMB, "The absolute free disk space, in MB, to maintain. When disk space falls below this threshold, new pods would be rejected. Default: 256") @@ -279,6 +210,6 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver") fs.BoolVar(&s.SerializeImagePulls, "serialize-image-pulls", s.SerializeImagePulls, "Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details. [default=true]") fs.BoolVar(&s.ExperimentalFlannelOverlay, "experimental-flannel-overlay", s.ExperimentalFlannelOverlay, "Experimental support for starting the kubelet with the default overlay network (flannel). Assumes flanneld is already running in client mode. [default=false]") - fs.DurationVar(&s.OutOfDiskTransitionFrequency, "outofdisk-transition-frequency", s.OutOfDiskTransitionFrequency, "Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status. Default: 5m0s") - fs.IPVar(&s.NodeIP, "node-ip", s.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node") + fs.DurationVar(&s.OutOfDiskTransitionFrequency.Duration, "outofdisk-transition-frequency", s.OutOfDiskTransitionFrequency.Duration, "Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status. Default: 5m0s") + fs.StringVar(&s.NodeIP, "node-ip", s.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node") } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 2010fb21e56..32d9599ad63 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -178,13 +178,13 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { } return &KubeletConfig{ - Address: s.Address, + Address: net.ParseIP(s.Address), AllowPrivileged: s.AllowPrivileged, Auth: nil, // default does not enforce auth[nz] CAdvisorInterface: nil, // launches background processes, not set here CgroupRoot: s.CgroupRoot, Cloud: nil, // cloud provider might start background processes - ClusterDNS: s.ClusterDNS, + ClusterDNS: net.ParseIP(s.ClusterDNS), ClusterDomain: s.ClusterDomain, ConfigFile: s.Config, ConfigureCBR0: s.ConfigureCBR0, @@ -199,12 +199,12 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { EnableServer: s.EnableServer, EventBurst: s.EventBurst, EventRecordQPS: s.EventRecordQPS, - FileCheckFrequency: s.FileCheckFrequency, + FileCheckFrequency: s.FileCheckFrequency.Duration, HostnameOverride: s.HostnameOverride, HostNetworkSources: hostNetworkSources, HostPIDSources: hostPIDSources, HostIPCSources: hostIPCSources, - HTTPCheckFrequency: s.HTTPCheckFrequency, + HTTPCheckFrequency: s.HTTPCheckFrequency.Duration, ImageGCPolicy: imageGCPolicy, KubeClient: nil, ManifestURL: s.ManifestURL, @@ -214,14 +214,14 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { MaxOpenFiles: s.MaxOpenFiles, MaxPerPodContainerCount: s.MaxPerPodContainerCount, MaxPods: s.MaxPods, - MinimumGCAge: s.MinimumGCAge, + MinimumGCAge: s.MinimumGCAge.Duration, Mounter: mounter, ChownRunner: chownRunner, ChmodRunner: chmodRunner, NetworkPluginName: s.NetworkPluginName, NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir), NodeLabels: s.NodeLabels, - NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency, + NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency.Duration, OOMAdjuster: oom.NewOOMAdjuster(), OSInterface: kubecontainer.RealOS{}, PodCIDR: s.PodCIDR, @@ -242,16 +242,16 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { Runonce: s.RunOnce, SerializeImagePulls: s.SerializeImagePulls, StandaloneMode: (len(s.APIServerList) == 0), - StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout, - SyncFrequency: s.SyncFrequency, + StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout.Duration, + SyncFrequency: s.SyncFrequency.Duration, SystemContainer: s.SystemContainer, TLSOptions: tlsOptions, Writer: writer, VolumePlugins: ProbeVolumePlugins(s.VolumePluginDir), - OutOfDiskTransitionFrequency: s.OutOfDiskTransitionFrequency, + OutOfDiskTransitionFrequency: s.OutOfDiskTransitionFrequency.Duration, ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay, - NodeIP: s.NodeIP, + NodeIP: net.ParseIP(s.NodeIP), }, nil } @@ -324,7 +324,7 @@ func Run(s *options.KubeletServer, kcfg *KubeletConfig) error { if s.HealthzPort > 0 { healthz.DefaultHealthz() go util.Until(func() { - err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress.String(), strconv.Itoa(s.HealthzPort)), nil) + err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(s.HealthzPort)), nil) if err != nil { glog.Errorf("Starting health server failed: %v", err) } diff --git a/contrib/mesos/pkg/executor/service/service.go b/contrib/mesos/pkg/executor/service/service.go index 048b59afde5..bf37c4d9ada 100644 --- a/contrib/mesos/pkg/executor/service/service.go +++ b/contrib/mesos/pkg/executor/service/service.go @@ -65,7 +65,7 @@ func NewKubeletExecutorServer() *KubeletExecutorServer { } else { k.RootDirectory = pwd // mesos sandbox dir } - k.Address = net.ParseIP(defaultBindingAddress()) + k.Address = defaultBindingAddress() return k } @@ -121,7 +121,7 @@ func (s *KubeletExecutorServer) runExecutor( dconfig := bindings.DriverConfig{ Executor: exec, HostnameOverride: s.HostnameOverride, - BindingAddress: s.Address, + BindingAddress: net.ParseIP(s.Address), } driver, err := bindings.NewMesosExecutorDriver(dconfig) if err != nil { diff --git a/contrib/mesos/pkg/minion/server.go b/contrib/mesos/pkg/minion/server.go index f141348c48c..7654b9d250a 100644 --- a/contrib/mesos/pkg/minion/server.go +++ b/contrib/mesos/pkg/minion/server.go @@ -135,7 +135,7 @@ func findMesosCgroup(prefix string) string { func (ms *MinionServer) launchProxyServer() { bindAddress := "0.0.0.0" if !ms.proxyBindall { - bindAddress = ms.KubeletExecutorServer.Address.String() + bindAddress = ms.KubeletExecutorServer.Address } args := []string{ fmt.Sprintf("--bind-address=%s", bindAddress), diff --git a/contrib/mesos/pkg/node/statusupdater.go b/contrib/mesos/pkg/node/statusupdater.go index 9644f8d422c..4f69656a0f7 100644 --- a/contrib/mesos/pkg/node/statusupdater.go +++ b/contrib/mesos/pkg/node/statusupdater.go @@ -51,7 +51,7 @@ func NewStatusUpdater(client *client.Client, relistPeriod time.Duration, nowFunc return &StatusUpdater{ client: client, relistPeriod: relistPeriod, - heartBeatPeriod: kubecfg.NodeStatusUpdateFrequency, + heartBeatPeriod: kubecfg.NodeStatusUpdateFrequency.Duration, nowFunc: nowFunc, } } diff --git a/contrib/mesos/pkg/node/statusupdater_test.go b/contrib/mesos/pkg/node/statusupdater_test.go index ffb0cf441fd..aba8fca5b33 100644 --- a/contrib/mesos/pkg/node/statusupdater_test.go +++ b/contrib/mesos/pkg/node/statusupdater_test.go @@ -48,7 +48,7 @@ func Test_nodeWithUpdatedStatus(t *testing.T) { cm := cmoptions.NewCMServer() kubecfg := kubeletoptions.NewKubeletServer() - assert.True(t, kubecfg.NodeStatusUpdateFrequency*3 < cm.NodeMonitorGracePeriod) // sanity check for defaults + assert.True(t, kubecfg.NodeStatusUpdateFrequency.Duration*3 < cm.NodeMonitorGracePeriod) // sanity check for defaults n := testNode(0, api.ConditionTrue, "KubeletReady") su := NewStatusUpdater(nil, cm.NodeMonitorPeriod, func() time.Time { return now }) @@ -63,12 +63,12 @@ func Test_nodeWithUpdatedStatus(t *testing.T) { assert.Equal(t, getCondition(&n2.Status, api.NodeReady).Reason, slaveReadyReason) assert.Equal(t, getCondition(&n2.Status, api.NodeReady).Message, slaveReadyMessage) - n = testNode(-kubecfg.NodeStatusUpdateFrequency, api.ConditionTrue, "KubeletReady") + n = testNode(-kubecfg.NodeStatusUpdateFrequency.Duration, api.ConditionTrue, "KubeletReady") n2, updated, err = su.nodeWithUpdatedStatus(n) assert.NoError(t, err) assert.False(t, updated, "no update expected b/c kubelet's update was missed only once") - n = testNode(-kubecfg.NodeStatusUpdateFrequency*3, api.ConditionTrue, "KubeletReady") + n = testNode(-kubecfg.NodeStatusUpdateFrequency.Duration*3, api.ConditionTrue, "KubeletReady") n2, updated, err = su.nodeWithUpdatedStatus(n) assert.NoError(t, err) assert.True(t, updated, "update expected b/c kubelet's update is older than 3*DefaultNodeStatusUpdateFrequency") diff --git a/docs/admin/kubelet.md b/docs/admin/kubelet.md index 2050a5575b6..4933807b40a 100644 --- a/docs/admin/kubelet.md +++ b/docs/admin/kubelet.md @@ -64,7 +64,7 @@ kubelet ### Options ``` - --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces) + --address="0.0.0.0": The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces) --allow-privileged[=false]: If true, allow containers to request privileged mode. [default=false] --api-servers=[]: List of Kubernetes API servers for publishing events, and reading pods and services. (ip:port), comma separated. --cadvisor-port=4194: The port of the localhost cAdvisor endpoint @@ -73,7 +73,7 @@ kubelet --chaos-chance=0: If > 0.0, introduce random client errors and latency. Intended for testing. [default=0.0] --cloud-config="": The path to the cloud provider configuration file. Empty string for no configuration file. --cloud-provider="": The provider for cloud services. Empty string for no provider. - --cluster-dns=: IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers + --cluster-dns="": IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers --cluster-domain="": Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains --config="": Path to the config file or directory of files --configure-cbr0[=false]: If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR. @@ -89,7 +89,7 @@ kubelet --experimental-flannel-overlay[=false]: Experimental support for starting the kubelet with the default overlay network (flannel). Assumes flanneld is already running in client mode. [default=false] --file-check-frequency=20s: Duration between checking config files for new data --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication. - --healthz-bind-address=127.0.0.1: The IP address for the healthz server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces) + --healthz-bind-address="127.0.0.1": The IP address for the healthz server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces) --healthz-port=10248: The port of the localhost healthz endpoint --host-ipc-sources="*": Comma-separated list of sources from which the Kubelet allows pods to use the host ipc namespace. [default="*"] --host-network-sources="*": Comma-separated list of sources from which the Kubelet allows pods to use of host network. [default="*"] @@ -114,8 +114,13 @@ kubelet --minimum-container-ttl-duration=1m0s: Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m' --network-plugin="": The name of the network plugin to be invoked for various events in kubelet/pod lifecycle --network-plugin-dir="/usr/libexec/kubernetes/kubelet-plugins/net/exec/": The full path of the directory in which to search for network plugins +<<<<<<< HEAD --node-ip=: IP address of the node. If set, kubelet will use this IP address for the node --node-labels=: Labels to add when registering the node in the cluster. Labels must are key=value pairs seperated by ','. +======= + --node-ip="": IP address of the node. If set, kubelet will use this IP address for the node + --node-labels="": Labels to add when registering the node in the cluster. Labels must be specified as a json map of key:value pairs. +>>>>>>> kubelet: move most of kubelet server configuration to a config object. --node-status-update-frequency=10s: Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s --oom-score-adj=-999: The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000] --outofdisk-transition-frequency=5m0s: Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status. Default: 5m0s diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 8c40d1bf44c..65591b40428 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -63,6 +63,219 @@ const ( ProxyModeIPTables ProxyMode = "iptables" ) +// TODO: curate the ordering and structure of this config object +type KubeletConfiguration struct { + // config is the path to the config file or directory of files + Config string `json:"config"` + // syncFrequency is the max period between synchronizing running + // containers and config + SyncFrequency unversioned.Duration `json:"syncFrequency"` + // fileCheckFrequency is the duration between checking config files for + // new data + FileCheckFrequency unversioned.Duration `json:"fileCheckFrequency"` + // httpCheckFrequency is the duration between checking http for new data + HTTPCheckFrequency unversioned.Duration `json:"httpCheckFrequency"` + // manifestURL is the URL for accessing the container manifest + ManifestURL string `json:"manifestURL"` + // manifestURLHeader is the HTTP header to use when accessing the manifest + // URL, with the key separated from the value with a ':', as in 'key:value' + ManifestURLHeader string `json:"manifestURLHeader"` + // enableServer enables the Kubelet's server + EnableServer bool `json:"enableServer"` + // address is the IP address for the Kubelet to serve on (set to 0.0.0.0 + // for all interfaces) + Address string `json:"address"` + // port is the port for the Kubelet to serve on. + Port uint `json:"port"` + // readOnlyPort is the read-only port for the Kubelet to serve on with + // no authentication/authorization (set to 0 to disable) + ReadOnlyPort uint `json:"readOnlyPort"` + // tLSCertFile is the file containing x509 Certificate for HTTPS. (CA cert, + // if any, concatenated after server cert). If tlsCertFile and + // tlsPrivateKeyFile are not provided, a self-signed certificate + // and key are generated for the public address and saved to the directory + // passed to certDir. + TLSCertFile string `json:"tLSCertFile"` + // tLSPrivateKeyFile is the ile containing x509 private key matching + // tlsCertFile. + TLSPrivateKeyFile string `json:"tLSPrivateKeyFile"` + // certDirectory is the directory where the TLS certs are located (by + // default /var/run/kubernetes). If tlsCertFile and tlsPrivateKeyFile + // are provided, this flag will be ignored. + CertDirectory string `json:"certDirectory"` + // hostnameOverride is the hostname used to identify the kubelet instead + // of the actual hostname. + HostnameOverride string `json:"hostnameOverride"` + // podInfraContainerImage is the image whose network/ipc namespaces + // containers in each pod will use. + PodInfraContainerImage string `json:"podInfraContainerImage"` + // dockerEndpoint is the path to the docker endpoint to communicate with. + DockerEndpoint string `json:"dockerEndpoint"` + // rootDirectory is the directory path to place kubelet files (volume + // mounts,etc). + RootDirectory string `json:"rootDirectory"` + // allowPrivileged enables containers to request privileged mode. + // Defaults to false. + AllowPrivileged bool `json:"allowPrivileged"` + // hostNetworkSources is a comma-separated list of sources from which the + // Kubelet allows pods to use of host network. Defaults to "*". + HostNetworkSources string `json:"hostNetworkSources"` + // hostPIDSources is a comma-separated list of sources from which the + // Kubelet allows pods to use the host pid namespace. Defaults to "*". + HostPIDSources string `json:"hostPIDSources"` + // hostIPCSources is a comma-separated list of sources from which the + // Kubelet allows pods to use the host ipc namespace. Defaults to "*". + HostIPCSources string `json:"hostIPCSources"` + // registryPullQPS is the limit of registry pulls per second. If 0, + // unlimited. Set to 0 for no limit. Defaults to 5.0. + RegistryPullQPS float64 `json:"registryPullQPS"` + // registryBurst is the maximum size of a bursty pulls, temporarily allows + // pulls to burst to this number, while still not exceeding registryQps. + // Only used if registryQps > 0. + RegistryBurst int `json:"registryBurst"` + // eventRecordQPS is the maximum event creations per second. If 0, there + // is no limit enforced. + EventRecordQPS float32 `json:"eventRecordQPS"` + // eventBurst is the maximum size of a bursty event records, temporarily + // allows event records to burst to this number, while still not exceeding + // event-qps. Only used if eventQps > 0 + EventBurst int `json:"eventBurst"` + // enableDebuggingHandlers enables server endpoints for log collection + // and local running of containers and commands + EnableDebuggingHandlers bool `json:"enableDebuggingHandlers"` + // minimumGCAge is the minimum age for a finished container before it is + // garbage collected. + MinimumGCAge unversioned.Duration `json:"minimumGCAge"` + // maxPerPodContainerCount is the maximum number of old instances to + // retain per container. Each container takes up some disk space. + MaxPerPodContainerCount int `json:"maxPerPodContainerCount"` + // maxContainerCount is the maximum number of old instances of containers + // to retain globally. Each container takes up some disk space. + MaxContainerCount int `json:"maxContainerCount"` + // cAdvisorPort is the port of the localhost cAdvisor endpoint + CAdvisorPort uint `json:"cAdvisorPort"` + // healthzPort is the port of the localhost healthz endpoint + HealthzPort int `json:"healthzPort"` + // healthzBindAddress is the IP address for the healthz server to serve + // on. + HealthzBindAddress string `json:"healthzBindAddress"` + // oomScoreAdj is The oom-score-adj value for kubelet process. Values + // must be within the range [-1000, 1000]. + OOMScoreAdj int `json:"oomScoreAdj"` + // registerNode enables automatic registration with the apiserver. + RegisterNode bool `json:"registerNode"` + // clusterDomain is the DNS domain for this cluster. If set, kubelet will + // configure all containers to search this domain in addition to the + // host's search domains. + ClusterDomain string `json:"clusterDomain"` + // masterServiceNamespace is The namespace from which the kubernetes + // master services should be injected into pods. + MasterServiceNamespace string `json:"masterServiceNamespace"` + // clusterDNS is the IP address for a cluster DNS server. If set, kubelet + // will configure all containers to use this for DNS resolution in + // addition to the host's DNS servers + ClusterDNS string `json:"clusterDNS"` + // streamingConnectionIdleTimeout is the maximum time a streaming connection + // can be idle before the connection is automatically closed. + StreamingConnectionIdleTimeout unversioned.Duration `json:"streamingConnectionIdleTimeout"` + // nodeStatusUpdateFrequency is the frequency that kubelet posts node + // status to master. Note: be cautious when changing the constant, it + // must work with nodeMonitorGracePeriod in nodecontroller. + NodeStatusUpdateFrequency unversioned.Duration `json:"nodeStatusUpdateFrequency"` + // imageGCHighThresholdPercent is the percent of disk usage after which + // image garbage collection is always run. + ImageGCHighThresholdPercent int `json:"imageGCHighThresholdPercent"` + // imageGCLowThresholdPercent is the percent of disk usage before which + // image garbage collection is never run. Lowest disk usage to garbage + // collect to. + ImageGCLowThresholdPercent int `json:"imageGCLowThresholdPercent"` + // lowDiskSpaceThresholdMB is the absolute free disk space, in MB, to + // maintain. When disk space falls below this threshold, new pods would + // be rejected. + LowDiskSpaceThresholdMB int `json:"lowDiskSpaceThresholdMB"` + // networkPluginName is the name of the network plugin to be invoked for + // various events in kubelet/pod lifecycle + NetworkPluginName string `json:"networkPluginName"` + // networkPluginDir is the full path of the directory in which to search + // for network plugins + NetworkPluginDir string `json:"networkPluginDir"` + // volumePluginDir is the full path of the directory in which to search + // for additional third party volume plugins + VolumePluginDir string `json:"volumePluginDir"` + // cloudProvider is the provider for cloud services. + CloudProvider string `json:"cloudProvider,omitempty"` + // cloudConfigFile is the path to the cloud provider configuration file. + CloudConfigFile string `json:"cloudConfigFile,omitempty"` + // resourceContainer is the absolute name of the resource-only container + // to create and run the Kubelet in. + ResourceContainer string `json:"resourceContainer,omitempty"` + // cgroupRoot is the root cgroup to use for pods. This is handled by the + // container runtime on a best effort basis. + CgroupRoot string `json:"cgroupRoot,omitempty"` + // containerRuntime is the container runtime to use. + ContainerRuntime string `json:"containerRuntime"` + // rktPath is hte path of rkt binary. Leave empty to use the first rkt in + // $PATH. + RktPath string `json:"rktPath,omitempty"` + // rktStage1Image is the image to use as stage1. Local paths and + // http/https URLs are supported. + RktStage1Image string `json:"rktStage1Image,omitempty"` + // systemContainer is the resource-only container in which to place + // all non-kernel processes that are not already in a container. Empty + // for no container. Rolling back the flag requires a reboot. + SystemContainer string `json:"systemContainer"` + // configureCBR0 enables the kublet to configure cbr0 based on + // Node.Spec.PodCIDR. + ConfigureCBR0 bool `json:"configureCbr0"` + // maxPods is the number of pods that can run on this Kubelet. + MaxPods int `json:"maxPods"` + // dockerExecHandlerName is the handler to use when executing a command + // in a container. Valid values are 'native' and 'nsenter'. Defaults to + // 'native'. + DockerExecHandlerName string `json:"dockerExecHandlerName"` + // The CIDR to use for pod IP addresses, only used in standalone mode. + // In cluster mode, this is obtained from the master. + PodCIDR string `json:"podCIDR"` + // ResolverConfig is the resolver configuration file used as the basis + // for the container DNS resolution configuration."), [] + ResolverConfig string `json:"resolvConf"` + // cpuCFSQuota is Enable CPU CFS quota enforcement for containers that + // specify CPU limits + CPUCFSQuota bool `json:"cpuCFSQuota"` + // containerized should be set to true if kubelet is running in a container. + Containerized bool `json:"containerized"` + // maxOpenFiles is Number of files that can be opened by Kubelet process. + MaxOpenFiles uint64 `json:"maxOpenFiles"` + // reconcileCIDR is Reconcile node CIDR with the CIDR specified by the + // API server. No-op if register-node or configure-cbr0 is false. + ReconcileCIDR bool `json:"reconcileCIDR"` + // registerSchedulable tells the kubelet to register the node as + // schedulable. No-op if register-node is false. + RegisterSchedulable bool `json:"registerSchedulable"` + // kubeAPIQPS is the QPS to use while talking with kubernetes apiserver + KubeAPIQPS float32 `json:"kubeAPIQPS"` + // kubeAPIBurst is the burst to allow while talking with kubernetes + // apiserver + KubeAPIBurst int `json:"kubeAPIBurst"` + // serializeImagePulls when enabled, tells the Kubelet to pull images one + // at a time. We recommend *not* changing the default value on nodes that + // run docker daemon with version < 1.9 or an Aufs storage backend. + // Issue #10959 has more details. + SerializeImagePulls bool `json:"serializeImagePulls"` + // experimentalFlannelOverlay enables experimental support for starting the + // kubelet with the default overlay network (flannel). Assumes flanneld + // is already running in client mode. + ExperimentalFlannelOverlay bool `json:"experimentalFlannelOverlay"` + // outOfDiskTransitionFrequency is duration for which the kubelet has to + // wait before transitioning out of out-of-disk node condition status. + OutOfDiskTransitionFrequency unversioned.Duration `json:"outOfDiskTransitionFrequency,omitempty"` + // nodeIP is IP address of the node. If set, kubelet will use this IP + // address for the node. + NodeIP string `json:"nodeIP,omitempty"` + // nodeLabels to add when registering the node in the cluster. + NodeLabels map[string]string `json:"nodeLabels"` +} + // LeaderElectionConfiguration defines the configuration of leader election // clients for components that can run with leader election enabled. type LeaderElectionConfiguration struct {