Split kubelet server initialization into two parts

First is initializing a KubeletConfig that starts no background
processes. Second is running the config. Provide a legacy path
that won't impact older callers while making it easier to customize
the interfaces passed to the Kubelet.

Used by OpenShift to inject some custom interfaces to the Kubelet
for config management.

cmd/hyperkube/kubelet.go

@@ -32,8 +32,8 @@ func NewKubelet() *Server {
 		queries Docker to see what is currently running.  It synchronizes the
 		configuration data, with the running set of containers by starting or stopping
 		Docker containers.`,
-		Run: func(_ *Server, args []string) error {
-			return s.Run(args)
+		Run: func(_ *Server, _ []string) error {
+			return s.Run(nil)
 		},
 	}
 	s.AddFlags(hks.Flags())

cmd/kubelet/app/server.go

@@ -254,66 +254,12 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.Containerized, "containerized", s.Containerized, "Experimental support for running kubelet in a container.  Intended for testing. [default=false]")
 }
 
-// Run runs the specified KubeletServer.  This should never exit.
-func (s *KubeletServer) Run(_ []string) error {
-	util.ReallyCrash = s.ReallyCrashForTesting
-	rand.Seed(time.Now().UTC().UnixNano())
-
-	// TODO(vmarmol): Do this through container config.
-	if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
-		glog.Warning(err)
-	}
-
-	var apiclient *client.Client
-	clientConfig, err := s.CreateAPIServerClientConfig()
-	if err == nil {
-		apiclient, err = client.New(clientConfig)
-	}
-	if err != nil && len(s.APIServerList) > 0 {
-		glog.Warningf("No API client: %v", err)
-	}
-
-	glog.V(2).Infof("Using root directory: %v", s.RootDirectory)
-
-	credentialprovider.SetPreferredDockercfgPath(s.RootDirectory)
-
-	cadvisorInterface, err := cadvisor.New(s.CadvisorPort)
-	if err != nil {
-		return err
-	}
-
-	imageGCPolicy := kubelet.ImageGCPolicy{
-		HighThresholdPercent: s.ImageGCHighThresholdPercent,
-		LowThresholdPercent:  s.ImageGCLowThresholdPercent,
-	}
-
-	diskSpacePolicy := kubelet.DiskSpacePolicy{
-		DockerFreeDiskMB: s.LowDiskSpaceThresholdMB,
-		RootFreeDiskMB:   s.LowDiskSpaceThresholdMB,
-	}
-	cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
-	if err != nil {
-		return err
-	}
-	glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
-
-	manifestURLHeader := make(http.Header)
-	if s.ManifestURLHeader != "" {
-		pieces := strings.Split(s.ManifestURLHeader, ":")
-		if len(pieces) != 2 {
-			return fmt.Errorf("manifest-url-header must have a single ':' key-value separator, got %q", s.ManifestURLHeader)
-		}
-		manifestURLHeader.Set(pieces[0], pieces[1])
-	}
-
+// KubeletConfig returns a KubeletConfig suitable for being run, or an error if the server setup
+// is not valid.  It will not start any background processes.
+func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) {
 	hostNetworkSources, err := kubelet.GetValidatedSources(strings.Split(s.HostNetworkSources, ","))
 	if err != nil {
-		return err
-	}
-
-	tlsOptions, err := s.InitializeTLS()
-	if err != nil {
-		return err
+		return nil, err
 	}
 
 	mounter := mount.New()
@@ -322,6 +268,11 @@ func (s *KubeletServer) Run(_ []string) error {
 		mounter = mount.NewNsenterMounter()
 	}
 
+	tlsOptions, err := s.InitializeTLS()
+	if err != nil {
+		return nil, err
+	}
+
 	var dockerExecHandler dockertools.ExecHandler
 	switch s.DockerExecHandlerName {
 	case "native":
@@ -333,7 +284,26 @@ func (s *KubeletServer) Run(_ []string) error {
 		dockerExecHandler = &dockertools.NativeExecHandler{}
 	}
 
-	kcfg := KubeletConfig{
+	imageGCPolicy := kubelet.ImageGCPolicy{
+		HighThresholdPercent: s.ImageGCHighThresholdPercent,
+		LowThresholdPercent:  s.ImageGCLowThresholdPercent,
+	}
+
+	diskSpacePolicy := kubelet.DiskSpacePolicy{
+		DockerFreeDiskMB: s.LowDiskSpaceThresholdMB,
+		RootFreeDiskMB:   s.LowDiskSpaceThresholdMB,
+	}
+
+	manifestURLHeader := make(http.Header)
+	if s.ManifestURLHeader != "" {
+		pieces := strings.Split(s.ManifestURLHeader, ":")
+		if len(pieces) != 2 {
+			return nil, fmt.Errorf("manifest-url-header must have a single ':' key-value separator, got %q", s.ManifestURLHeader)
+		}
+		manifestURLHeader.Set(pieces[0], pieces[1])
+	}
+
+	return &KubeletConfig{
 		Address:                        s.Address,
 		AllowPrivileged:                s.AllowPrivileged,
 		HostNetworkSources:             hostNetworkSources,
@@ -358,11 +328,11 @@ func (s *KubeletServer) Run(_ []string) error {
 		Runonce:                        s.RunOnce,
 		Port:                           s.Port,
 		ReadOnlyPort:                   s.ReadOnlyPort,
-		CadvisorInterface:              cadvisorInterface,
+		CadvisorInterface:              nil, // launches background processes, not set here
 		EnableServer:                   s.EnableServer,
 		EnableDebuggingHandlers:        s.EnableDebuggingHandlers,
 		DockerClient:                   dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
-		KubeClient:                     apiclient,
+		KubeClient:                     nil,
 		MasterServiceNamespace:         s.MasterServiceNamespace,
 		VolumePlugins:                  ProbeVolumePlugins(),
 		NetworkPlugins:                 ProbeNetworkPlugins(s.NetworkPluginDir),
@@ -371,7 +341,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		TLSOptions:                     tlsOptions,
 		ImageGCPolicy:                  imageGCPolicy,
 		DiskSpacePolicy:                diskSpacePolicy,
-		Cloud:                          cloud,
+		Cloud:                          nil, // cloud provider might start background processes
 		NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency,
 		ResourceContainer:         s.ResourceContainer,
 		CgroupRoot:                s.CgroupRoot,
@@ -383,9 +353,58 @@ func (s *KubeletServer) Run(_ []string) error {
 		PodCIDR:                   s.PodCIDR,
 		MaxPods:                   s.MaxPods,
 		DockerExecHandler:         dockerExecHandler,
+	}, nil
+}
+
+// Run runs the specified KubeletServer for the given KubeletConfig.  This should never exit.
+// The kcfg argument may be nil - if so, it is initialized from the settings on KubeletServer.
+// Otherwise, the caller is assumed to have set up the KubeletConfig object and all defaults
+// will be ignored.
+func (s *KubeletServer) Run(kcfg *KubeletConfig) error {
+	if kcfg == nil {
+		cfg, err := s.KubeletConfig()
+		if err != nil {
+			return err
+		}
+		kcfg = cfg
+
+		clientConfig, err := s.CreateAPIServerClientConfig()
+		if err == nil {
+			kcfg.KubeClient, err = client.New(clientConfig)
+		}
+		if err != nil && len(s.APIServerList) > 0 {
+			glog.Warningf("No API client: %v", err)
+		}
+
+		cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
+		if err != nil {
+			return err
+		}
+		glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
+		kcfg.Cloud = cloud
 	}
 
-	if err := RunKubelet(&kcfg, nil); err != nil {
+	if kcfg.CadvisorInterface == nil {
+		ca, err := cadvisor.New(s.CadvisorPort)
+		if err != nil {
+			return err
+		}
+		kcfg.CadvisorInterface = ca
+	}
+
+	util.ReallyCrash = s.ReallyCrashForTesting
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	credentialprovider.SetPreferredDockercfgPath(s.RootDirectory)
+
+	glog.V(2).Infof("Using root directory: %v", s.RootDirectory)
+
+	// TODO(vmarmol): Do this through container config.
+	if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
+		glog.Warning(err)
+	}
+
+	if err := RunKubelet(kcfg, nil); err != nil {
 		return err
 	}
 

cmd/kubelet/kubelet.go

@@ -43,7 +43,7 @@ func main() {
 
 	verflag.PrintAndExitIfRequested()
 
-	if err := s.Run(pflag.CommandLine.Args()); err != nil {
+	if err := s.Run(nil); err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}