diff --git a/pkg/kubelet/container_manager_linux.go b/pkg/kubelet/container_manager_linux.go index 0f3576b5d4b..0eea05845ce 100644 --- a/pkg/kubelet/container_manager_linux.go +++ b/pkg/kubelet/container_manager_linux.go @@ -35,7 +35,9 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/errors" + "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/oom" + "k8s.io/kubernetes/pkg/util/sets" ) const ( @@ -70,75 +72,62 @@ func newSystemContainer(containerName string) *systemContainer { } } +type nodeConfig struct { + dockerDaemonContainerName string + systemContainerName string + kubeletContainerName string +} + type containerManagerImpl struct { + cadvisorInterface cadvisor.Interface + mountUtil mount.Interface + nodeConfig // External containers being managed. systemContainers []*systemContainer } var _ containerManager = &containerManagerImpl{} +// checks if the required cgroups subsystems are mounted. +// As of now, only 'cpu' and 'memory' are required. +func validateSystemRequirements(mountUtil mount.Interface) error { + const ( + cgroupMountType = "cgroup" + localErr = "system validation failed" + ) + mountPoints, err := mountUtil.List() + if err != nil { + return fmt.Errorf("%s - %v", localErr, err) + } + expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory") + for _, mountPoint := range mountPoints { + if mountPoint.Type == cgroupMountType { + for _, opt := range mountPoint.Opts { + if expectedCgroups.Has(opt) { + expectedCgroups.Delete(opt) + } + } + } + } + + if expectedCgroups.Len() > 0 { + return fmt.Errorf("%s - Following Cgroup subsystem not mounted: %v", localErr, expectedCgroups.List()) + } + return nil +} + // TODO(vmarmol): Add limits to the system containers. // Takes the absolute name of the specified containers. // Empty container name disables use of the specified container. -func newContainerManager(cadvisorInterface cadvisor.Interface, dockerDaemonContainerName, systemContainerName, kubeletContainerName string) (containerManager, error) { - systemContainers := []*systemContainer{} - - if dockerDaemonContainerName != "" { - cont := newSystemContainer(dockerDaemonContainerName) - - info, err := cadvisorInterface.MachineInfo() - var capacity = api.ResourceList{} - if err != nil { - } else { - capacity = CapacityFromMachineInfo(info) - } - memoryLimit := (int64(capacity.Memory().Value() * DockerMemoryLimitThresholdPercent / 100)) - if memoryLimit < MinDockerMemoryLimit { - glog.Warningf("Memory limit %d for container %s is too small, reset it to %d", memoryLimit, dockerDaemonContainerName, MinDockerMemoryLimit) - memoryLimit = MinDockerMemoryLimit - } - - glog.V(2).Infof("Configure resource-only container %s with memory limit: %d", dockerDaemonContainerName, memoryLimit) - - dockerContainer := &fs.Manager{ - Cgroups: &configs.Cgroup{ - Name: dockerDaemonContainerName, - Memory: memoryLimit, - MemorySwap: -1, - AllowAllDevices: true, - }, - } - cont.ensureStateFunc = func(manager *fs.Manager) error { - return ensureDockerInContainer(cadvisorInterface, -900, dockerContainer) - } - systemContainers = append(systemContainers, cont) - } - - if systemContainerName != "" { - if systemContainerName == "/" { - return nil, fmt.Errorf("system container cannot be root (\"/\")") - } - - rootContainer := &fs.Manager{ - Cgroups: &configs.Cgroup{ - Name: "/", - }, - } - manager := createManager(systemContainerName) - - err := ensureSystemContainer(rootContainer, manager) - if err != nil { - return nil, err - } - systemContainers = append(systemContainers, newSystemContainer(systemContainerName)) - } - - if kubeletContainerName != "" { - systemContainers = append(systemContainers, newSystemContainer(kubeletContainerName)) - } - +func newContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, dockerDaemonContainerName, systemContainerName, kubeletContainerName string) (containerManager, error) { return &containerManagerImpl{ - systemContainers: systemContainers, + cadvisorInterface: cadvisorInterface, + mountUtil: mountUtil, + nodeConfig: nodeConfig{ + dockerDaemonContainerName: dockerDaemonContainerName, + systemContainerName: systemContainerName, + kubeletContainerName: kubeletContainerName, + }, }, nil } @@ -152,7 +141,73 @@ func createManager(containerName string) *fs.Manager { } } +func (cm *containerManagerImpl) setupNode() error { + if err := validateSystemRequirements(cm.mountUtil); err != nil { + return err + } + systemContainers := []*systemContainer{} + if cm.dockerDaemonContainerName != "" { + cont := newSystemContainer(cm.dockerDaemonContainerName) + + info, err := cm.cadvisorInterface.MachineInfo() + var capacity = api.ResourceList{} + if err != nil { + } else { + capacity = CapacityFromMachineInfo(info) + } + memoryLimit := (int64(capacity.Memory().Value() * DockerMemoryLimitThresholdPercent / 100)) + if memoryLimit < MinDockerMemoryLimit { + glog.Warningf("Memory limit %d for container %s is too small, reset it to %d", memoryLimit, cm.dockerDaemonContainerName, MinDockerMemoryLimit) + memoryLimit = MinDockerMemoryLimit + } + + glog.V(2).Infof("Configure resource-only container %s with memory limit: %d", cm.dockerDaemonContainerName, memoryLimit) + + dockerContainer := &fs.Manager{ + Cgroups: &configs.Cgroup{ + Name: cm.dockerDaemonContainerName, + Memory: memoryLimit, + MemorySwap: -1, + AllowAllDevices: true, + }, + } + cont.ensureStateFunc = func(manager *fs.Manager) error { + return ensureDockerInContainer(cm.cadvisorInterface, -900, dockerContainer) + } + systemContainers = append(systemContainers, cont) + } + + if cm.systemContainerName != "" { + if cm.systemContainerName == "/" { + return fmt.Errorf("system container cannot be root (\"/\")") + } + + rootContainer := &fs.Manager{ + Cgroups: &configs.Cgroup{ + Name: "/", + }, + } + manager := createManager(cm.systemContainerName) + + err := ensureSystemContainer(rootContainer, manager) + if err != nil { + return err + } + systemContainers = append(systemContainers, newSystemContainer(cm.systemContainerName)) + } + + if cm.kubeletContainerName != "" { + systemContainers = append(systemContainers, newSystemContainer(cm.kubeletContainerName)) + } + cm.systemContainers = systemContainers + return nil +} + func (cm *containerManagerImpl) Start() error { + // Setup the node + if err := cm.setupNode(); err != nil { + return err + } // Don't run a background thread if there are no ensureStateFuncs. numEnsureStateFuncs := 0 for _, cont := range cm.systemContainers { diff --git a/pkg/kubelet/container_manager_linux_test.go b/pkg/kubelet/container_manager_linux_test.go new file mode 100644 index 00000000000..a0bd899e298 --- /dev/null +++ b/pkg/kubelet/container_manager_linux_test.go @@ -0,0 +1,125 @@ +// +build linux + +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "k8s.io/kubernetes/pkg/util/mount" +) + +type fakeMountInterface struct { + mountPoints []mount.MountPoint +} + +func (mi *fakeMountInterface) Mount(source string, target string, fstype string, options []string) error { + return fmt.Errorf("unsupported") +} + +func (mi *fakeMountInterface) Unmount(target string) error { + return fmt.Errorf("unsupported") +} + +func (mi *fakeMountInterface) List() ([]mount.MountPoint, error) { + return mi.mountPoints, nil +} + +func (mi *fakeMountInterface) IsLikelyNotMountPoint(file string) (bool, error) { + return false, fmt.Errorf("unsupported") +} + +func fakeContainerMgrMountInt() mount.Interface { + return &fakeMountInterface{ + []mount.MountPoint{ + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpuset"}, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpu"}, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpuacct"}, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "memory"}, + }, + }, + } +} + +func TestCgroupMountValidationSuccess(t *testing.T) { + assert.Nil(t, validateSystemRequirements(fakeContainerMgrMountInt())) +} + +func TestCgroupMountValidationMemoryMissing(t *testing.T) { + mountInt := &fakeMountInterface{ + []mount.MountPoint{ + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpuset"}, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpu"}, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpuacct"}, + }, + }, + } + assert.Error(t, validateSystemRequirements(mountInt)) +} + +func TestCgroupMountValidationMultipleSubsytem(t *testing.T) { + mountInt := &fakeMountInterface{ + []mount.MountPoint{ + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpuset", "memory"}, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpu"}, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpuacct"}, + }, + }, + } + assert.Nil(t, validateSystemRequirements(mountInt)) +} diff --git a/pkg/kubelet/container_manager_unsupported.go b/pkg/kubelet/container_manager_unsupported.go index 8444852d0a0..c18e6799735 100644 --- a/pkg/kubelet/container_manager_unsupported.go +++ b/pkg/kubelet/container_manager_unsupported.go @@ -23,6 +23,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/util/mount" ) type unsupportedContainerManager struct { @@ -38,6 +39,6 @@ func (unsupportedContainerManager) SystemContainersLimit() api.ResourceList { return api.ResourceList{} } -func newContainerManager(cadvisorInterface cadvisor.Interface, dockerDaemonContainer, systemContainer, kubeletContainer string) (containerManager, error) { +func newContainerManager(mounter mount.Interface, cadvisorInterface cadvisor.Interface, dockerDaemonContainer, systemContainer, kubeletContainer string) (containerManager, error) { return &unsupportedContainerManager{}, nil } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 165bfd5a0fd..851eb1a52e9 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -362,7 +362,7 @@ func NewMainKubelet( // Setup container manager, can fail if the devices hierarchy is not mounted // (it is required by Docker however). - containerManager, err := newContainerManager(cadvisorInterface, dockerDaemonContainer, systemContainer, resourceContainer) + containerManager, err := newContainerManager(mounter, cadvisorInterface, dockerDaemonContainer, systemContainer, resourceContainer) if err != nil { return nil, fmt.Errorf("failed to create the Container Manager: %v", err) } @@ -758,6 +758,7 @@ func (kl *Kubelet) Run(updates <-chan PodUpdate) { // Move Kubelet to a container. if kl.resourceContainer != "" { + // Fixme: I need to reside inside ContainerManager interface. err := util.RunInResourceContainer(kl.resourceContainer) if err != nil { glog.Warningf("Failed to move Kubelet to container %q: %v", kl.resourceContainer, err) diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index dc3548b44a8..861285d25ed 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -131,7 +131,7 @@ func newTestKubelet(t *testing.T) *TestKubelet { t: t, } kubelet.volumeManager = newVolumeManager() - kubelet.containerManager, _ = newContainerManager(mockCadvisor, "", "", "") + kubelet.containerManager, _ = newContainerManager(fakeContainerMgrMountInt(), mockCadvisor, "", "", "") kubelet.networkConfigured = true fakeClock := &util.FakeClock{Time: time.Now()} kubelet.backOff = util.NewBackOff(time.Second, time.Minute) diff --git a/pkg/kubelet/runonce_test.go b/pkg/kubelet/runonce_test.go index 5b3fc3dccdc..5d36d708ddd 100644 --- a/pkg/kubelet/runonce_test.go +++ b/pkg/kubelet/runonce_test.go @@ -51,7 +51,7 @@ func TestRunOnce(t *testing.T) { diskSpaceManager: diskSpaceManager, containerRuntime: fakeRuntime, } - kb.containerManager, _ = newContainerManager(cadvisor, "", "", "") + kb.containerManager, _ = newContainerManager(fakeContainerMgrMountInt(), cadvisor, "", "", "") kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", network.NewFakeHost(nil)) if err := kb.setupDataDirs(); err != nil {