From c54ba12faad4642cff589b5f87f76f37b1158f02 Mon Sep 17 00:00:00 2001 From: Vishnu kannan Date: Thu, 3 Mar 2016 16:37:09 -0800 Subject: [PATCH] Update node status to include the absense of cpu hardcapping. Signed-off-by: Vishnu kannan --- pkg/kubelet/cm/container_manager.go | 8 +++ pkg/kubelet/cm/container_manager_linux.go | 51 ++++++++++++++++--- .../cm/container_manager_linux_test.go | 45 ++++++++++++++-- pkg/kubelet/cm/container_manager_stub.go | 4 ++ .../cm/container_manager_unsupported.go | 4 ++ pkg/kubelet/kubelet.go | 6 +++ 6 files changed, 109 insertions(+), 9 deletions(-) diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 9ee4ef42908..e18bc6865e2 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -33,6 +33,9 @@ type ContainerManager interface { // Returns a NodeConfig that is being used by the container manager. GetNodeConfig() NodeConfig + + // Returns internal Status. + Status() Status } type NodeConfig struct { @@ -41,3 +44,8 @@ type NodeConfig struct { KubeletCgroupsName string ContainerRuntime string } + +type Status struct { + // Any soft requirements that were unsatisfied. + SoftRequirements error +} diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 7220d04faa4..37b9376fa8a 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "os/exec" + "path" "strconv" "strings" "sync" @@ -34,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/util" utilerrors "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/oom" @@ -79,24 +81,35 @@ type containerManagerImpl struct { cadvisorInterface cadvisor.Interface mountUtil mount.Interface NodeConfig + status Status // External containers being managed. systemContainers []*systemContainer periodicTasks []func() } +type features struct { + cpuHardcapping bool +} + var _ ContainerManager = &containerManagerImpl{} // checks if the required cgroups subsystems are mounted. // As of now, only 'cpu' and 'memory' are required. -func validateSystemRequirements(mountUtil mount.Interface) error { +// cpu quota is a soft requirement. +func validateSystemRequirements(mountUtil mount.Interface) (features, error) { const ( cgroupMountType = "cgroup" localErr = "system validation failed" ) + var ( + cpuMountPoint string + f features + ) mountPoints, err := mountUtil.List() if err != nil { - return fmt.Errorf("%s - %v", localErr, err) + return f, fmt.Errorf("%s - %v", localErr, err) } + expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory") for _, mountPoint := range mountPoints { if mountPoint.Type == cgroupMountType { @@ -104,14 +117,31 @@ func validateSystemRequirements(mountUtil mount.Interface) error { if expectedCgroups.Has(opt) { expectedCgroups.Delete(opt) } + if opt == "cpu" { + cpuMountPoint = mountPoint.Path + } } } } if expectedCgroups.Len() > 0 { - return fmt.Errorf("%s - Following Cgroup subsystem not mounted: %v", localErr, expectedCgroups.List()) + return f, fmt.Errorf("%s - Following Cgroup subsystem not mounted: %v", localErr, expectedCgroups.List()) } - return nil + + // Check if cpu quota is available. + // CPU cgroup is required and so it expected to be mounted at this point. + periodExists, err := util.FileExists(path.Join(cpuMountPoint, "cpu.cfs_period_us")) + if err != nil { + glog.Errorf("failed to detect if CPU cgroup cpu.cfs_period_us is available - %v", err) + } + quotaExists, err := util.FileExists(path.Join(cpuMountPoint, "cpu.cfs_quota_us")) + if err != nil { + glog.Errorf("failed to detect if CPU cgroup cpu.cfs_quota_us is available - %v", err) + } + if quotaExists && periodExists { + f.cpuHardcapping = true + } + return f, nil } // TODO(vmarmol): Add limits to the system containers. @@ -185,10 +215,13 @@ func setupKernelTunables(option KernelTunableBehavior) error { } func (cm *containerManagerImpl) setupNode() error { - if err := validateSystemRequirements(cm.mountUtil); err != nil { + f, err := validateSystemRequirements(cm.mountUtil) + if err != nil { return err } - + if !f.cpuHardcapping { + cm.status.SoftRequirements = fmt.Errorf("CPU hardcapping unsupported") + } // TODO: plumb kernel tunable options into container manager, right now, we modify by default if err := setupKernelTunables(KernelTunableModify); err != nil { return err @@ -312,6 +345,12 @@ func (cm *containerManagerImpl) GetNodeConfig() NodeConfig { return cm.NodeConfig } +func (cm *containerManagerImpl) Status() Status { + cm.RLock() + defer cm.RUnlock() + return cm.status +} + func (cm *containerManagerImpl) Start() error { // Setup the node if err := cm.setupNode(); err != nil { diff --git a/pkg/kubelet/cm/container_manager_linux_test.go b/pkg/kubelet/cm/container_manager_linux_test.go index 66c508eb16d..34e91c83b69 100644 --- a/pkg/kubelet/cm/container_manager_linux_test.go +++ b/pkg/kubelet/cm/container_manager_linux_test.go @@ -20,9 +20,13 @@ package cm import ( "fmt" + "io/ioutil" + "os" + "path" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "k8s.io/kubernetes/pkg/util/mount" ) @@ -75,7 +79,9 @@ func fakeContainerMgrMountInt() mount.Interface { } func TestCgroupMountValidationSuccess(t *testing.T) { - assert.Nil(t, validateSystemRequirements(fakeContainerMgrMountInt())) + f, err := validateSystemRequirements(fakeContainerMgrMountInt()) + assert.Nil(t, err) + assert.False(t, f.cpuHardcapping, "cpu hardcapping is expected to be disabled") } func TestCgroupMountValidationMemoryMissing(t *testing.T) { @@ -98,7 +104,8 @@ func TestCgroupMountValidationMemoryMissing(t *testing.T) { }, }, } - assert.Error(t, validateSystemRequirements(mountInt)) + _, err := validateSystemRequirements(mountInt) + assert.Error(t, err) } func TestCgroupMountValidationMultipleSubsytem(t *testing.T) { @@ -121,5 +128,37 @@ func TestCgroupMountValidationMultipleSubsytem(t *testing.T) { }, }, } - assert.Nil(t, validateSystemRequirements(mountInt)) + _, err := validateSystemRequirements(mountInt) + assert.Nil(t, err) +} + +func TestSoftRequirementsValidationSuccess(t *testing.T) { + req := require.New(t) + tempDir, err := ioutil.TempDir("", "") + req.NoError(err) + req.NoError(ioutil.WriteFile(path.Join(tempDir, "cpu.cfs_period_us"), []byte("0"), os.ModePerm)) + req.NoError(ioutil.WriteFile(path.Join(tempDir, "cpu.cfs_quota_us"), []byte("0"), os.ModePerm)) + mountInt := &fakeMountInterface{ + []mount.MountPoint{ + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpuset"}, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpu"}, + Path: tempDir, + }, + { + Device: "cgroup", + Type: "cgroup", + Opts: []string{"rw", "relatime", "cpuacct", "memory"}, + }, + }, + } + f, err := validateSystemRequirements(mountInt) + assert.NoError(t, err) + assert.True(t, f.cpuHardcapping, "cpu hardcapping is expected to be enabled") } diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index c79ae885083..4bca506c2fa 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -38,6 +38,10 @@ func (cm *containerManagerStub) GetNodeConfig() NodeConfig { return NodeConfig{} } +func (cm *containerManagerStub) Status() Status { + return Status{} +} + func NewStubContainerManager() ContainerManager { return &containerManagerStub{} } diff --git a/pkg/kubelet/cm/container_manager_unsupported.go b/pkg/kubelet/cm/container_manager_unsupported.go index 99114004d2d..426c95ca4cc 100644 --- a/pkg/kubelet/cm/container_manager_unsupported.go +++ b/pkg/kubelet/cm/container_manager_unsupported.go @@ -43,6 +43,10 @@ func (unsupportedContainerManager) GetNodeConfig() NodeConfig { return NodeConfig{} } +func (cm *unsupportedContainerManager) Status() Status { + return Status{} +} + func NewContainerManager(_ mount.Interface, _ cadvisor.Interface, _ NodeConfig) (ContainerManager, error) { return &unsupportedContainerManager{}, nil } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 49a9b8458ef..cb6a2e73333 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -2931,6 +2931,12 @@ func (kl *Kubelet) setNodeReadyCondition(node *api.Node) { } } + // Record any soft requirements that were not met in the container manager. + status := kl.containerManager.Status() + if status.SoftRequirements != nil { + newNodeReadyCondition.Message = fmt.Sprintf("%s. WARNING: %s", newNodeReadyCondition.Message, status.SoftRequirements.Error()) + } + readyConditionUpdated := false needToRecordEvent := false for i := range node.Status.Conditions {