diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index ed839dedaf4..64a284195d0 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -102,6 +102,7 @@ type KubeletServer struct { ResourceContainer string CgroupRoot string ContainerRuntime string + DockerDaemonContainer string // Flags intended for testing @@ -158,6 +159,7 @@ func NewKubeletServer() *KubeletServer { ResourceContainer: "/kubelet", CgroupRoot: "/", ContainerRuntime: "docker", + DockerDaemonContainer: "/docker-daemon", } } @@ -212,6 +214,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.ResourceContainer, "resource-container", s.ResourceContainer, "Absolute name of the resource-only container to create and run the Kubelet in (Default: /kubelet).") fs.StringVar(&s.CgroupRoot, "cgroup_root", s.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '/', which means top-level.") fs.StringVar(&s.ContainerRuntime, "container_runtime", s.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.") + fs.StringVar(&s.DockerDaemonContainer, "docker-daemon-container", s.DockerDaemonContainer, "Optional resource-only container in which to place the Docker Daemon. Empty for no container (Default: /docker-daemon).") // Flags intended for testing, not recommended used in production environments. fs.BoolVar(&s.ReallyCrashForTesting, "really-crash-for-testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.") @@ -321,6 +324,7 @@ func (s *KubeletServer) Run(_ []string) error { CgroupRoot: s.CgroupRoot, ContainerRuntime: s.ContainerRuntime, Mounter: mounter, + DockerDaemonContainer: s.DockerDaemonContainer, } RunKubelet(&kcfg, nil) @@ -432,6 +436,7 @@ func SimpleKubelet(client *client.Client, CgroupRoot: "/", ContainerRuntime: "docker", Mounter: mount.New(), + DockerDaemonContainer: "/docker-daemon", } return &kcfg } @@ -562,6 +567,7 @@ type KubeletConfig struct { CgroupRoot string ContainerRuntime string Mounter mount.Interface + DockerDaemonContainer string } func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) { @@ -609,7 +615,8 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.OSInterface, kc.CgroupRoot, kc.ContainerRuntime, - kc.Mounter) + kc.Mounter, + kc.DockerDaemonContainer) if err != nil { return nil, nil, err diff --git a/pkg/kubelet/container_manager.go b/pkg/kubelet/container_manager.go new file mode 100644 index 00000000000..84b20de18da --- /dev/null +++ b/pkg/kubelet/container_manager.go @@ -0,0 +1,24 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +// Manages the containers running on a machine. +type containerManager interface { + // Runs the container manager's housekeeping. + // - Ensures that the Docker daemon is in a container. + Start() error +} diff --git a/pkg/kubelet/container_manager_linux.go b/pkg/kubelet/container_manager_linux.go new file mode 100644 index 00000000000..292d2a8351f --- /dev/null +++ b/pkg/kubelet/container_manager_linux.go @@ -0,0 +1,120 @@ +// +build linux + +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors" + "github.com/docker/libcontainer/cgroups" + "github.com/docker/libcontainer/cgroups/fs" + "github.com/docker/libcontainer/configs" + "github.com/golang/glog" +) + +type containerManagerImpl struct { + // Absolute name of the desired container that Docker should be in. + dockerContainerName string + + // The manager of the resource-only container Docker should be in. + manager fs.Manager +} + +var _ containerManager = &containerManagerImpl{} + +// Takes the absolute name that the Docker daemon should be in. +// Empty container name disables moving the Docker daemon. +func newContainerManager(dockerDaemonContainer string) (containerManager, error) { + return &containerManagerImpl{ + dockerContainerName: dockerDaemonContainer, + manager: fs.Manager{ + Cgroups: &configs.Cgroup{ + Name: dockerDaemonContainer, + AllowAllDevices: true, + }, + }, + }, nil +} + +func (cm *containerManagerImpl) Start() error { + if cm.dockerContainerName != "" { + go util.Until(func() { + err := cm.ensureDockerInContainer() + if err != nil { + glog.Warningf("[ContainerManager] Failed to ensure Docker is in a container: %v", err) + } + }, time.Minute, util.NeverStop) + } + return nil +} + +// Ensures that the Docker daemon is in the desired container. +func (cm *containerManagerImpl) ensureDockerInContainer() error { + // What container is Docker in? + out, err := exec.Command("pidof", "docker").Output() + if err != nil { + return fmt.Errorf("failed to find pid of Docker container: %v", err) + } + + // The output of pidof is a list of pids. + // Docker may be forking and thus there would be more than one result. + pids := []int{} + for _, pidStr := range strings.Split(strings.TrimSpace(string(out)), " ") { + pid, err := strconv.Atoi(pidStr) + if err != nil { + continue + } + pids = append(pids, pid) + } + + // Move if the pid is not already in the desired container. + errs := []error{} + for _, pid := range pids { + cont, err := getContainer(pid) + if err != nil { + errs = append(errs, fmt.Errorf("failed to find container of PID %q: %v", pid, err)) + } + + if cont != cm.dockerContainerName { + err = cm.manager.Apply(pid) + if err != nil { + errs = append(errs, fmt.Errorf("failed to move PID %q (in %q) to %q", pid, cont, cm.dockerContainerName)) + } + } + } + + return errors.NewAggregate(errs) +} + +// Gets the (CPU) container the specified pid is in. +func getContainer(pid int) (string, error) { + f, err := os.Open(fmt.Sprintf("/proc/%d/cgroup", pid)) + if err != nil { + return "", err + } + defer f.Close() + + return cgroups.ParseCgroupFile("cpu", f) +} diff --git a/pkg/kubelet/container_manager_unsupported.go b/pkg/kubelet/container_manager_unsupported.go new file mode 100644 index 00000000000..6c543e1e617 --- /dev/null +++ b/pkg/kubelet/container_manager_unsupported.go @@ -0,0 +1,36 @@ +// +build !linux + +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" +) + +type unsupportedContainerManager struct { +} + +var _ containerManager = &unsupportedContainerManager{} + +func (unsupportedContainerManager) Start() error { + return fmt.Errorf("Container Manager is unsupported in this build") +} + +func newContainerManager(dockerDaemonContainer string) (containerManager, error) { + return &unsupportedContainerManager{}, nil +} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 0cbdb8e5207..9e84ccbb8ca 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -136,7 +136,8 @@ func NewMainKubelet( osInterface kubecontainer.OSInterface, cgroupRoot string, containerRuntime string, - mounter mount.Interface) (*Kubelet, error) { + mounter mount.Interface, + dockerDaemonContainer string) (*Kubelet, error) { if rootDirectory == "" { return nil, fmt.Errorf("invalid root directory %q", rootDirectory) } @@ -276,10 +277,19 @@ func NewMainKubelet( return nil, err } klet.containerRuntime = rktRuntime + + // No Docker daemon to put in a container. + dockerDaemonContainer = "" default: return nil, fmt.Errorf("unsupported container runtime %q specified", containerRuntime) } + containerManager, err := newContainerManager(dockerDaemonContainer) + if err != nil { + return nil, fmt.Errorf("failed to create the Container Manager: %v", err) + } + klet.containerManager = containerManager + // Wait for the runtime to be up with a timeout. if err := waitUntilRuntimeIsUp(klet.containerRuntime, maxWaitForContainerRuntime); err != nil { return nil, fmt.Errorf("timed out waiting for %q to come up: %v", containerRuntime, err) @@ -434,6 +444,9 @@ type Kubelet struct { // Mounter to use for volumes. mounter mount.Interface + + // Manager of non-Runtime containers. + containerManager containerManager } // getRootDir returns the full path to the directory under which kubelet can @@ -624,10 +637,16 @@ func (kl *Kubelet) Run(updates <-chan PodUpdate) { err := kl.imageManager.Start() if err != nil { - kl.recorder.Eventf(kl.nodeRef, "imageManagerFailed", "Failed to start ImageManager %v", err) + kl.recorder.Eventf(kl.nodeRef, "kubeletSetupFailed", "Failed to start ImageManager %v", err) glog.Errorf("Failed to start ImageManager, images may not be garbage collected: %v", err) } + err = kl.containerManager.Start() + if err != nil { + kl.recorder.Eventf(kl.nodeRef, "kubeletSetupFailed", "Failed to start ContainerManager %v", err) + glog.Errorf("Failed to start ContainerManager, system may not be properly isolated: %v", err) + } + go util.Until(kl.updateRuntimeUp, 5*time.Second, util.NeverStop) go kl.syncNodeStatus() // Run the system oom watcher forever.