From bb79365e2aa0c38673d1e7f31f74d04982965820 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 1 Jul 2015 15:02:30 -0400 Subject: [PATCH] Split kubelet server initialization into two parts First is initializing a KubeletConfig that starts no background processes. Second is running the config. Provide a legacy path that won't impact older callers while making it easier to customize the interfaces passed to the Kubelet. Used by OpenShift to inject some custom interfaces to the Kubelet for config management. --- cmd/hyperkube/kubelet.go | 4 +- cmd/kubelet/app/server.go | 145 +++++++++++++++++++++----------------- cmd/kubelet/kubelet.go | 2 +- 3 files changed, 85 insertions(+), 66 deletions(-) diff --git a/cmd/hyperkube/kubelet.go b/cmd/hyperkube/kubelet.go index f93eac02c20..e12b20db555 100644 --- a/cmd/hyperkube/kubelet.go +++ b/cmd/hyperkube/kubelet.go @@ -32,8 +32,8 @@ func NewKubelet() *Server { queries Docker to see what is currently running. It synchronizes the configuration data, with the running set of containers by starting or stopping Docker containers.`, - Run: func(_ *Server, args []string) error { - return s.Run(args) + Run: func(_ *Server, _ []string) error { + return s.Run(nil) }, } s.AddFlags(hks.Flags()) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 8cbdcfba15c..5d13bb71340 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -254,66 +254,12 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.Containerized, "containerized", s.Containerized, "Experimental support for running kubelet in a container. Intended for testing. [default=false]") } -// Run runs the specified KubeletServer. This should never exit. -func (s *KubeletServer) Run(_ []string) error { - util.ReallyCrash = s.ReallyCrashForTesting - rand.Seed(time.Now().UTC().UnixNano()) - - // TODO(vmarmol): Do this through container config. - if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil { - glog.Warning(err) - } - - var apiclient *client.Client - clientConfig, err := s.CreateAPIServerClientConfig() - if err == nil { - apiclient, err = client.New(clientConfig) - } - if err != nil && len(s.APIServerList) > 0 { - glog.Warningf("No API client: %v", err) - } - - glog.V(2).Infof("Using root directory: %v", s.RootDirectory) - - credentialprovider.SetPreferredDockercfgPath(s.RootDirectory) - - cadvisorInterface, err := cadvisor.New(s.CadvisorPort) - if err != nil { - return err - } - - imageGCPolicy := kubelet.ImageGCPolicy{ - HighThresholdPercent: s.ImageGCHighThresholdPercent, - LowThresholdPercent: s.ImageGCLowThresholdPercent, - } - - diskSpacePolicy := kubelet.DiskSpacePolicy{ - DockerFreeDiskMB: s.LowDiskSpaceThresholdMB, - RootFreeDiskMB: s.LowDiskSpaceThresholdMB, - } - cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) - if err != nil { - return err - } - glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) - - manifestURLHeader := make(http.Header) - if s.ManifestURLHeader != "" { - pieces := strings.Split(s.ManifestURLHeader, ":") - if len(pieces) != 2 { - return fmt.Errorf("manifest-url-header must have a single ':' key-value separator, got %q", s.ManifestURLHeader) - } - manifestURLHeader.Set(pieces[0], pieces[1]) - } - +// KubeletConfig returns a KubeletConfig suitable for being run, or an error if the server setup +// is not valid. It will not start any background processes. +func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) { hostNetworkSources, err := kubelet.GetValidatedSources(strings.Split(s.HostNetworkSources, ",")) if err != nil { - return err - } - - tlsOptions, err := s.InitializeTLS() - if err != nil { - return err + return nil, err } mounter := mount.New() @@ -322,6 +268,11 @@ func (s *KubeletServer) Run(_ []string) error { mounter = mount.NewNsenterMounter() } + tlsOptions, err := s.InitializeTLS() + if err != nil { + return nil, err + } + var dockerExecHandler dockertools.ExecHandler switch s.DockerExecHandlerName { case "native": @@ -333,7 +284,26 @@ func (s *KubeletServer) Run(_ []string) error { dockerExecHandler = &dockertools.NativeExecHandler{} } - kcfg := KubeletConfig{ + imageGCPolicy := kubelet.ImageGCPolicy{ + HighThresholdPercent: s.ImageGCHighThresholdPercent, + LowThresholdPercent: s.ImageGCLowThresholdPercent, + } + + diskSpacePolicy := kubelet.DiskSpacePolicy{ + DockerFreeDiskMB: s.LowDiskSpaceThresholdMB, + RootFreeDiskMB: s.LowDiskSpaceThresholdMB, + } + + manifestURLHeader := make(http.Header) + if s.ManifestURLHeader != "" { + pieces := strings.Split(s.ManifestURLHeader, ":") + if len(pieces) != 2 { + return nil, fmt.Errorf("manifest-url-header must have a single ':' key-value separator, got %q", s.ManifestURLHeader) + } + manifestURLHeader.Set(pieces[0], pieces[1]) + } + + return &KubeletConfig{ Address: s.Address, AllowPrivileged: s.AllowPrivileged, HostNetworkSources: hostNetworkSources, @@ -358,11 +328,11 @@ func (s *KubeletServer) Run(_ []string) error { Runonce: s.RunOnce, Port: s.Port, ReadOnlyPort: s.ReadOnlyPort, - CadvisorInterface: cadvisorInterface, + CadvisorInterface: nil, // launches background processes, not set here EnableServer: s.EnableServer, EnableDebuggingHandlers: s.EnableDebuggingHandlers, DockerClient: dockertools.ConnectToDockerOrDie(s.DockerEndpoint), - KubeClient: apiclient, + KubeClient: nil, MasterServiceNamespace: s.MasterServiceNamespace, VolumePlugins: ProbeVolumePlugins(), NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir), @@ -371,7 +341,7 @@ func (s *KubeletServer) Run(_ []string) error { TLSOptions: tlsOptions, ImageGCPolicy: imageGCPolicy, DiskSpacePolicy: diskSpacePolicy, - Cloud: cloud, + Cloud: nil, // cloud provider might start background processes NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency, ResourceContainer: s.ResourceContainer, CgroupRoot: s.CgroupRoot, @@ -383,9 +353,58 @@ func (s *KubeletServer) Run(_ []string) error { PodCIDR: s.PodCIDR, MaxPods: s.MaxPods, DockerExecHandler: dockerExecHandler, + }, nil +} + +// Run runs the specified KubeletServer for the given KubeletConfig. This should never exit. +// The kcfg argument may be nil - if so, it is initialized from the settings on KubeletServer. +// Otherwise, the caller is assumed to have set up the KubeletConfig object and all defaults +// will be ignored. +func (s *KubeletServer) Run(kcfg *KubeletConfig) error { + if kcfg == nil { + cfg, err := s.KubeletConfig() + if err != nil { + return err + } + kcfg = cfg + + clientConfig, err := s.CreateAPIServerClientConfig() + if err == nil { + kcfg.KubeClient, err = client.New(clientConfig) + } + if err != nil && len(s.APIServerList) > 0 { + glog.Warningf("No API client: %v", err) + } + + cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) + if err != nil { + return err + } + glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) + kcfg.Cloud = cloud } - if err := RunKubelet(&kcfg, nil); err != nil { + if kcfg.CadvisorInterface == nil { + ca, err := cadvisor.New(s.CadvisorPort) + if err != nil { + return err + } + kcfg.CadvisorInterface = ca + } + + util.ReallyCrash = s.ReallyCrashForTesting + rand.Seed(time.Now().UTC().UnixNano()) + + credentialprovider.SetPreferredDockercfgPath(s.RootDirectory) + + glog.V(2).Infof("Using root directory: %v", s.RootDirectory) + + // TODO(vmarmol): Do this through container config. + if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil { + glog.Warning(err) + } + + if err := RunKubelet(kcfg, nil); err != nil { return err } diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index dac193ee32d..2160efc9a32 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -43,7 +43,7 @@ func main() { verflag.PrintAndExitIfRequested() - if err := s.Run(pflag.CommandLine.Args()); err != nil { + if err := s.Run(nil); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) }