Merge pull request #6813 from vmarmol/in-container

Run Kubelet and Kube-proxy in resource-only containers.

cmd/kube-proxy/app/server.go

@@ -44,6 +44,7 @@ type ProxyServer struct {
 	HealthzPort        int
 	HealthzBindAddress util.IP
 	OOMScoreAdj        int
+	ResourceContainer  string
 }
 
 // NewProxyServer creates a new ProxyServer object with default parameters
@@ -53,6 +54,7 @@ func NewProxyServer() *ProxyServer {
 		HealthzPort:        10249,
 		HealthzBindAddress: util.IP(net.ParseIP("127.0.0.1")),
 		OOMScoreAdj:        -899,
+		ResourceContainer:  "/kube-proxy",
 	}
 }
 
@@ -63,14 +65,23 @@ func (s *ProxyServer) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.HealthzPort, "healthz_port", s.HealthzPort, "The port to bind the health check server. Use 0 to disable.")
 	fs.Var(&s.HealthzBindAddress, "healthz_bind_address", "The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)")
 	fs.IntVar(&s.OOMScoreAdj, "oom_score_adj", s.OOMScoreAdj, "The oom_score_adj value for kube-proxy process. Values must be within the range [-1000, 1000]")
+	fs.StringVar(&s.ResourceContainer, "resource_container", s.ResourceContainer, "Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).")
 }
 
 // Run runs the specified ProxyServer.  This should never exit.
 func (s *ProxyServer) Run(_ []string) error {
+	// TODO(vmarmol): Use container config for this.
 	if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
 		glog.Info(err)
 	}
 
+	// Run in its own container.
+	if err := util.RunInResourceContainer(s.ResourceContainer); err != nil {
+		glog.Warningf("Failed to start in resource-only container %q: %v", s.ResourceContainer, err)
+	} else {
+		glog.Infof("Running in resource-only container %q", s.ResourceContainer)
+	}
+
 	serviceConfig := config.NewServiceConfig()
 	endpointsConfig := config.NewEndpointsConfig()
 

cmd/kubelet/app/server.go

@@ -96,6 +96,7 @@ type KubeletServer struct {
 	TLSPrivateKeyFile              string
 	CertDirectory                  string
 	NodeStatusUpdateFrequency      time.Duration
+	ResourceContainer              string
 
 	// Flags intended for testing
 
@@ -147,6 +148,7 @@ func NewKubeletServer() *KubeletServer {
 		HostNetworkSources:          kubelet.FileSource,
 		CertDirectory:               "/var/run/kubernetes",
 		NodeStatusUpdateFrequency:   10 * time.Second,
+		ResourceContainer:           "/kubelet",
 	}
 }
 
@@ -196,6 +198,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.NetworkPluginName, "network_plugin", s.NetworkPluginName, "<Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle")
 	fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
 	fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
+	fs.StringVar(&s.ResourceContainer, "resource_container", s.ResourceContainer, "Absolute name of the resource-only container to create and run the Kubelet in (Default: /kubelet).")
 
 	// Flags intended for testing, not recommended used in production environments.
 	fs.BoolVar(&s.ReallyCrashForTesting, "really_crash_for_testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.")
@@ -207,6 +210,7 @@ func (s *KubeletServer) Run(_ []string) error {
 	util.ReallyCrash = s.ReallyCrashForTesting
 	rand.Seed(time.Now().UTC().UnixNano())
 
+	// TODO(vmarmol): Do this through container config.
 	if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
 		glog.Info(err)
 	}
@@ -293,6 +297,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		ImageGCPolicy:                  imageGCPolicy,
 		Cloud:                          cloud,
 		NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency,
+		ResourceContainer:         s.ResourceContainer,
 	}
 
 	RunKubelet(&kcfg, nil)
@@ -412,6 +417,7 @@ func SimpleKubelet(client *client.Client,
 		ImageGCPolicy:           imageGCPolicy,
 		Cloud:                   cloud,
 		NodeStatusUpdateFrequency: 10 * time.Second,
+		ResourceContainer:         "/kubelet",
 	}
 	return &kcfg
 }
@@ -534,6 +540,7 @@ type KubeletConfig struct {
 	ImageGCPolicy                  kubelet.ImageGCPolicy
 	Cloud                          cloudprovider.Interface
 	NodeStatusUpdateFrequency      time.Duration
+	ResourceContainer              string
 }
 
 func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) {
@@ -576,7 +583,8 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.CadvisorInterface,
 		kc.ImageGCPolicy,
 		kc.Cloud,
-		kc.NodeStatusUpdateFrequency)
+		kc.NodeStatusUpdateFrequency,
+		kc.ResourceContainer)
 
 	if err != nil {
 		return nil, nil, err

pkg/kubelet/kubelet.go

@@ -122,7 +122,8 @@ func NewMainKubelet(
 	cadvisorInterface cadvisor.Interface,
 	imageGCPolicy ImageGCPolicy,
 	cloud cloudprovider.Interface,
-	nodeStatusUpdateFrequency time.Duration) (*Kubelet, error) {
+	nodeStatusUpdateFrequency time.Duration,
+	resourceContainer string) (*Kubelet, error) {
 	if rootDirectory == "" {
 		return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
 	}
@@ -228,6 +229,7 @@ func NewMainKubelet(
 		nodeRef:                        nodeRef,
 		containerManager:               containerManager,
 		nodeStatusUpdateFrequency:      nodeStatusUpdateFrequency,
+		resourceContainer:              resourceContainer,
 	}
 
 	klet.podManager = newBasicPodManager(klet.kubeClient)
@@ -358,6 +360,10 @@ type Kubelet struct {
 	//    status. Kubelet may fail to update node status reliablly if the value is too small,
 	//    as it takes time to gather all necessary node information.
 	nodeStatusUpdateFrequency time.Duration
+
+	// The name of the resource-only container to run the Kubelet in (empty for no container).
+	// Name must be absolute.
+	resourceContainer string
 }
 
 // getRootDir returns the full path to the directory under which kubelet can
@@ -536,6 +542,16 @@ func (kl *Kubelet) Run(updates <-chan PodUpdate) {
 	if kl.kubeClient == nil {
 		glog.Warning("No api server defined - no node status update will be sent.")
 	}
+
+	// Move Kubelet to a container.
+	if kl.resourceContainer != "" {
+		err := util.RunInResourceContainer(kl.resourceContainer)
+		if err != nil {
+			glog.Warningf("Failed to move Kubelet to container %q: %v", kl.resourceContainer, err)
+		}
+		glog.Infof("Running in container %q", kl.resourceContainer)
+	}
+
 	go kl.syncNodeStatus()
 	kl.statusManager.Start()
 	kl.syncLoop(updates, kl)

pkg/util/resource_container_linux.go

@@ -0,0 +1,41 @@
+// +build linux
+
+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"os"
+
+	"github.com/docker/libcontainer/cgroups/fs"
+	"github.com/docker/libcontainer/configs"
+)
+
+// Creates resource-only containerName if it does not already exist and moves
+// the current process to it.
+//
+// containerName must be an absolute container name.
+func RunInResourceContainer(containerName string) error {
+	manager := fs.Manager{
+		Cgroups: &configs.Cgroup{
+			Name:            containerName,
+			AllowAllDevices: true,
+		},
+	}
+
+	return manager.Apply(os.Getpid())
+}

pkg/util/resource_container_unsupported.go

@@ -0,0 +1,27 @@
+// +build !linux
+
+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"errors"
+)
+
+func RunInResourceContainer(containerName string) error {
+	return errors.New("resource-only containers unsupported in this platform")
+}