Merge pull request #8122 from vmarmol/docker-container

Run Docker Daemon in a Resource-only Container

cmd/kubelet/app/server.go

@@ -102,6 +102,7 @@ type KubeletServer struct {
 	ResourceContainer              string
 	CgroupRoot                     string
 	ContainerRuntime               string
+	DockerDaemonContainer          string
 
 	// Flags intended for testing
 
@@ -158,6 +159,7 @@ func NewKubeletServer() *KubeletServer {
 		ResourceContainer:           "/kubelet",
 		CgroupRoot:                  "/",
 		ContainerRuntime:            "docker",
+		DockerDaemonContainer:       "/docker-daemon",
 	}
 }
 
@@ -212,6 +214,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ResourceContainer, "resource-container", s.ResourceContainer, "Absolute name of the resource-only container to create and run the Kubelet in (Default: /kubelet).")
 	fs.StringVar(&s.CgroupRoot, "cgroup_root", s.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '/', which means top-level.")
 	fs.StringVar(&s.ContainerRuntime, "container_runtime", s.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.")
+	fs.StringVar(&s.DockerDaemonContainer, "docker-daemon-container", s.DockerDaemonContainer, "Optional resource-only container in which to place the Docker Daemon. Empty for no container (Default: /docker-daemon).")
 
 	// Flags intended for testing, not recommended used in production environments.
 	fs.BoolVar(&s.ReallyCrashForTesting, "really-crash-for-testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.")
@@ -321,6 +324,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		CgroupRoot:                s.CgroupRoot,
 		ContainerRuntime:          s.ContainerRuntime,
 		Mounter:                   mounter,
+		DockerDaemonContainer:     s.DockerDaemonContainer,
 	}
 
 	RunKubelet(&kcfg, nil)
@@ -432,6 +436,7 @@ func SimpleKubelet(client *client.Client,
 		CgroupRoot:                "/",
 		ContainerRuntime:          "docker",
 		Mounter:                   mount.New(),
+		DockerDaemonContainer:     "/docker-daemon",
 	}
 	return &kcfg
 }
@@ -562,6 +567,7 @@ type KubeletConfig struct {
 	CgroupRoot                     string
 	ContainerRuntime               string
 	Mounter                        mount.Interface
+	DockerDaemonContainer          string
 }
 
 func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) {
@@ -609,7 +615,8 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.OSInterface,
 		kc.CgroupRoot,
 		kc.ContainerRuntime,
-		kc.Mounter)
+		kc.Mounter,
+		kc.DockerDaemonContainer)
 
 	if err != nil {
 		return nil, nil, err

pkg/kubelet/container_manager.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+// Manages the containers running on a machine.
+type containerManager interface {
+	// Runs the container manager's housekeeping.
+	// - Ensures that the Docker daemon is in a container.
+	Start() error
+}

pkg/kubelet/container_manager_linux.go

@@ -0,0 +1,120 @@
+// +build linux
+
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors"
+	"github.com/docker/libcontainer/cgroups"
+	"github.com/docker/libcontainer/cgroups/fs"
+	"github.com/docker/libcontainer/configs"
+	"github.com/golang/glog"
+)
+
+type containerManagerImpl struct {
+	// Absolute name of the desired container that Docker should be in.
+	dockerContainerName string
+
+	// The manager of the resource-only container Docker should be in.
+	manager fs.Manager
+}
+
+var _ containerManager = &containerManagerImpl{}
+
+// Takes the absolute name that the Docker daemon should be in.
+// Empty container name disables moving the Docker daemon.
+func newContainerManager(dockerDaemonContainer string) (containerManager, error) {
+	return &containerManagerImpl{
+		dockerContainerName: dockerDaemonContainer,
+		manager: fs.Manager{
+			Cgroups: &configs.Cgroup{
+				Name:            dockerDaemonContainer,
+				AllowAllDevices: true,
+			},
+		},
+	}, nil
+}
+
+func (cm *containerManagerImpl) Start() error {
+	if cm.dockerContainerName != "" {
+		go util.Until(func() {
+			err := cm.ensureDockerInContainer()
+			if err != nil {
+				glog.Warningf("[ContainerManager] Failed to ensure Docker is in a container: %v", err)
+			}
+		}, time.Minute, util.NeverStop)
+	}
+	return nil
+}
+
+// Ensures that the Docker daemon is in the desired container.
+func (cm *containerManagerImpl) ensureDockerInContainer() error {
+	// What container is Docker in?
+	out, err := exec.Command("pidof", "docker").Output()
+	if err != nil {
+		return fmt.Errorf("failed to find pid of Docker container: %v", err)
+	}
+
+	// The output of pidof is a list of pids.
+	// Docker may be forking and thus there would be more than one result.
+	pids := []int{}
+	for _, pidStr := range strings.Split(strings.TrimSpace(string(out)), " ") {
+		pid, err := strconv.Atoi(pidStr)
+		if err != nil {
+			continue
+		}
+		pids = append(pids, pid)
+	}
+
+	// Move if the pid is not already in the desired container.
+	errs := []error{}
+	for _, pid := range pids {
+		cont, err := getContainer(pid)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to find container of PID %q: %v", pid, err))
+		}
+
+		if cont != cm.dockerContainerName {
+			err = cm.manager.Apply(pid)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("failed to move PID %q (in %q) to %q", pid, cont, cm.dockerContainerName))
+			}
+		}
+	}
+
+	return errors.NewAggregate(errs)
+}
+
+// Gets the (CPU) container the specified pid is in.
+func getContainer(pid int) (string, error) {
+	f, err := os.Open(fmt.Sprintf("/proc/%d/cgroup", pid))
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	return cgroups.ParseCgroupFile("cpu", f)
+}

pkg/kubelet/container_manager_unsupported.go

@@ -0,0 +1,36 @@
+// +build !linux
+
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"fmt"
+)
+
+type unsupportedContainerManager struct {
+}
+
+var _ containerManager = &unsupportedContainerManager{}
+
+func (unsupportedContainerManager) Start() error {
+	return fmt.Errorf("Container Manager is unsupported in this build")
+}
+
+func newContainerManager(dockerDaemonContainer string) (containerManager, error) {
+	return &unsupportedContainerManager{}, nil
+}

pkg/kubelet/kubelet.go

@@ -136,7 +136,8 @@ func NewMainKubelet(
 	osInterface kubecontainer.OSInterface,
 	cgroupRoot string,
 	containerRuntime string,
-	mounter mount.Interface) (*Kubelet, error) {
+	mounter mount.Interface,
+	dockerDaemonContainer string) (*Kubelet, error) {
 	if rootDirectory == "" {
 		return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
 	}
@@ -276,10 +277,19 @@ func NewMainKubelet(
 			return nil, err
 		}
 		klet.containerRuntime = rktRuntime
+
+		// No Docker daemon to put in a container.
+		dockerDaemonContainer = ""
 	default:
 		return nil, fmt.Errorf("unsupported container runtime %q specified", containerRuntime)
 	}
 
+	containerManager, err := newContainerManager(dockerDaemonContainer)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the Container Manager: %v", err)
+	}
+	klet.containerManager = containerManager
+
 	// Wait for the runtime to be up with a timeout.
 	if err := waitUntilRuntimeIsUp(klet.containerRuntime, maxWaitForContainerRuntime); err != nil {
 		return nil, fmt.Errorf("timed out waiting for %q to come up: %v", containerRuntime, err)
@@ -434,6 +444,9 @@ type Kubelet struct {
 
 	// Mounter to use for volumes.
 	mounter mount.Interface
+
+	// Manager of non-Runtime containers.
+	containerManager containerManager
 }
 
 // getRootDir returns the full path to the directory under which kubelet can
@@ -624,10 +637,16 @@ func (kl *Kubelet) Run(updates <-chan PodUpdate) {
 
 	err := kl.imageManager.Start()
 	if err != nil {
-		kl.recorder.Eventf(kl.nodeRef, "imageManagerFailed", "Failed to start ImageManager %v", err)
+		kl.recorder.Eventf(kl.nodeRef, "kubeletSetupFailed", "Failed to start ImageManager %v", err)
 		glog.Errorf("Failed to start ImageManager, images may not be garbage collected: %v", err)
 	}
 
+	err = kl.containerManager.Start()
+	if err != nil {
+		kl.recorder.Eventf(kl.nodeRef, "kubeletSetupFailed", "Failed to start ContainerManager %v", err)
+		glog.Errorf("Failed to start ContainerManager, system may not be properly isolated: %v", err)
+	}
+
 	go util.Until(kl.updateRuntimeUp, 5*time.Second, util.NeverStop)
 	go kl.syncNodeStatus()
 	// Run the system oom watcher forever.