diff --git a/pkg/kubelet/container_manager_linux.go b/pkg/kubelet/container_manager_linux.go index b595c4b3d3e..362227db4a7 100644 --- a/pkg/kubelet/container_manager_linux.go +++ b/pkg/kubelet/container_manager_linux.go @@ -28,6 +28,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource" + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors" "github.com/docker/libcontainer/cgroups" @@ -36,6 +37,15 @@ import ( "github.com/golang/glog" ) +const ( + // The percent of the machine memory capacity. The value is used to calculate + // docker memory resource container's hardlimit to workaround docker memory + // leakage issue. Please see kubernetes/issues/9881 for more detail. + DockerMemoryLimitThresholdPercent = 70 + // The minimum memory limit allocated to docker container: 150Mi + MinDockerMemoryLimit = 150 * 1024 * 1024 +) + // A non-user container tracked by the Kubelet. type systemContainer struct { // Absolute name of the container. @@ -69,13 +79,36 @@ var _ containerManager = &containerManagerImpl{} // TODO(vmarmol): Add limits to the system containers. // Takes the absolute name of the specified containers. // Empty container name disables use of the specified container. -func newContainerManager(dockerDaemonContainerName, systemContainerName, kubeletContainerName string) (containerManager, error) { +func newContainerManager(cadvisorInterface cadvisor.Interface, dockerDaemonContainerName, systemContainerName, kubeletContainerName string) (containerManager, error) { systemContainers := []*systemContainer{} if dockerDaemonContainerName != "" { cont := newSystemContainer(dockerDaemonContainerName) + + info, err := cadvisorInterface.MachineInfo() + var capacity = api.ResourceList{} + if err != nil { + } else { + capacity = CapacityFromMachineInfo(info) + } + memoryLimit := (int64(capacity.Memory().Value() * DockerMemoryLimitThresholdPercent / 100)) + if memoryLimit < MinDockerMemoryLimit { + glog.Warningf("Memory limit %d for container %s is too small, reset it to %d", memoryLimit, dockerDaemonContainerName, MinDockerMemoryLimit) + memoryLimit = MinDockerMemoryLimit + } + + glog.V(2).Infof("Configure resource-only container %s with memory limit: %d", dockerDaemonContainerName, memoryLimit) + + dockerContainer := &fs.Manager{ + Cgroups: &configs.Cgroup{ + Name: dockerDaemonContainerName, + Memory: memoryLimit, + MemorySwap: -1, + AllowAllDevices: true, + }, + } cont.ensureStateFunc = func(manager *fs.Manager) error { - return ensureDockerInContainer(-900, createManager(dockerDaemonContainerName)) + return ensureDockerInContainer(cadvisorInterface, -900, dockerContainer) } systemContainers = append(systemContainers, cont) } @@ -162,7 +195,7 @@ func (cm *containerManagerImpl) SystemContainersLimit() api.ResourceList { } // Ensures that the Docker daemon is in the desired container. -func ensureDockerInContainer(oomScoreAdj int, manager *fs.Manager) error { +func ensureDockerInContainer(cadvisor cadvisor.Interface, oomScoreAdj int, manager *fs.Manager) error { // What container is Docker in? out, err := exec.Command("pidof", "docker").Output() if err != nil { diff --git a/pkg/kubelet/container_manager_unsupported.go b/pkg/kubelet/container_manager_unsupported.go index 8a14b331f92..23f7bf2ce85 100644 --- a/pkg/kubelet/container_manager_unsupported.go +++ b/pkg/kubelet/container_manager_unsupported.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor" ) type unsupportedContainerManager struct { @@ -37,6 +38,6 @@ func (unsupportedContainerManager) SystemContainersLimit() api.ResourceList { return api.ResourceList{} } -func newContainerManager(dockerDaemonContainer, systemContainer, kubeletContainer string) (containerManager, error) { +func newContainerManager(cadvisorInterface cadvisor.Interface, dockerDaemonContainer, systemContainer, kubeletContainer string) (containerManager, error) { return &unsupportedContainerManager{}, nil } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index bfc2f3d31d3..5d422690494 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -309,7 +309,7 @@ func NewMainKubelet( // Setup container manager, can fail if the devices hierarchy is not mounted // (it is required by Docker however). - containerManager, err := newContainerManager(dockerDaemonContainer, systemContainer, resourceContainer) + containerManager, err := newContainerManager(cadvisorInterface, dockerDaemonContainer, systemContainer, resourceContainer) if err != nil { return nil, fmt.Errorf("failed to create the Container Manager: %v", err) } diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 5a9a90470e0..07268943b01 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -133,7 +133,7 @@ func newTestKubelet(t *testing.T) *TestKubelet { t: t, } kubelet.volumeManager = newVolumeManager() - kubelet.containerManager, _ = newContainerManager("", "", "") + kubelet.containerManager, _ = newContainerManager(mockCadvisor, "", "", "") return &TestKubelet{kubelet, fakeDocker, mockCadvisor, fakeKubeClient, fakeMirrorClient} } @@ -243,7 +243,7 @@ func newTestKubeletWithFakeRuntime(t *testing.T) *TestKubeletWithFakeRuntime { t: t, } kubelet.volumeManager = newVolumeManager() - kubelet.containerManager, _ = newContainerManager("", "", "") + kubelet.containerManager, _ = newContainerManager(mockCadvisor, "", "", "") return &TestKubeletWithFakeRuntime{kubelet, fakeRuntime, mockCadvisor, fakeKubeClient, fakeMirrorClient} } diff --git a/pkg/kubelet/runonce_test.go b/pkg/kubelet/runonce_test.go index 9eba420ad69..23429cf104e 100644 --- a/pkg/kubelet/runonce_test.go +++ b/pkg/kubelet/runonce_test.go @@ -89,7 +89,7 @@ func TestRunOnce(t *testing.T) { os: kubecontainer.FakeOS{}, volumeManager: newVolumeManager(), } - kb.containerManager, _ = newContainerManager("", "", "") + kb.containerManager, _ = newContainerManager(cadvisor, "", "", "") kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", network.NewFakeHost(nil)) if err := kb.setupDataDirs(); err != nil {