Add pod-level metric for CPU and memory stats

This PR adds the pod-level metrics for CPU and memory stats. cAdvisor
can get all pod cgroup information so we can add this pod-level CPU and
memory stats information from the corresponding pod cgroup

pkg/kubelet/apis/stats/v1alpha1/types.go

@@ -86,6 +86,12 @@ type PodStats struct {
 	// +patchMergeKey=name
 	// +patchStrategy=merge
 	Containers []ContainerStats `json:"containers" patchStrategy:"merge" patchMergeKey:"name"`
+	// Stats pertaining to CPU resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead).
+	// +optional
+	CPU *CPUStats `json:"cpu,omitempty"`
+	// Stats pertaining to memory (RAM) resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead).
+	// +optional
+	Memory *MemoryStats `json:"memory,omitempty"`
 	// Stats pertaining to network resources.
 	// +optional
 	Network *NetworkStats `json:"network,omitempty"`

pkg/kubelet/cm/cgroup_manager_linux.go

@@ -45,6 +45,8 @@ const (
 	libcontainerCgroupfs libcontainerCgroupManagerType = "cgroupfs"
 	// libcontainerSystemd means use libcontainer with systemd
 	libcontainerSystemd libcontainerCgroupManagerType = "systemd"
+	// systemdSuffix is the cgroup name suffix for systemd
+	systemdSuffix string = ".slice"
 )
 
 // hugePageSizeList is useful for converting to the hugetlb canonical unit
@@ -68,8 +70,8 @@ func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) st
 			}
 			// detect if we are given a systemd style name.
 			// if so, we do not want to do double encoding.
-			if strings.HasSuffix(part, ".slice") {
+			if IsSystemdStyleName(part) {
-				part = strings.TrimSuffix(part, ".slice")
+				part = strings.TrimSuffix(part, systemdSuffix)
 				separatorIndex := strings.LastIndex(part, "-")
 				if separatorIndex >= 0 && separatorIndex < len(part) {
 					part = part[separatorIndex+1:]
@@ -87,8 +89,8 @@ func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) st
 		result = "-"
 	}
 	// always have a .slice suffix
-	if !strings.HasSuffix(result, ".slice") {
+	if !IsSystemdStyleName(result) {
-		result = result + ".slice"
+		result = result + systemdSuffix
 	}
 
 	// if the caller desired the result in cgroupfs format...
@@ -114,6 +116,13 @@ func ConvertCgroupFsNameToSystemd(cgroupfsName string) (string, error) {
 	return path.Base(cgroupfsName), nil
 }
 
+func IsSystemdStyleName(name string) bool {
+	if strings.HasSuffix(name, systemdSuffix) {
+		return true
+	}
+	return false
+}
+
 // libcontainerAdapter provides a simplified interface to libcontainer based on libcontainer type.
 type libcontainerAdapter struct {
 	// cgroupManagerType defines how to interface with libcontainer
@@ -151,15 +160,18 @@ func (l *libcontainerAdapter) revertName(name string) CgroupName {
 	if l.cgroupManagerType != libcontainerSystemd {
 		return CgroupName(name)
 	}
+	return CgroupName(RevertFromSystemdToCgroupStyleName(name))
+}
 
+func RevertFromSystemdToCgroupStyleName(name string) string {
 	driverName, err := ConvertCgroupFsNameToSystemd(name)
 	if err != nil {
 		panic(err)
 	}
-	driverName = strings.TrimSuffix(driverName, ".slice")
+	driverName = strings.TrimSuffix(driverName, systemdSuffix)
 	driverName = strings.Replace(driverName, "-", "/", -1)
 	driverName = strings.Replace(driverName, "_", "-", -1)
-	return CgroupName(driverName)
+	return driverName
 }
 
 // adaptName converts a CgroupName identifier to a driver specific conversion value.

pkg/kubelet/cm/cgroup_manager_unsupported.go

@@ -77,3 +77,11 @@ func ConvertCgroupFsNameToSystemd(cgroupfsName string) (string, error) {
 func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) string {
 	return ""
 }
+
+func RevertFromSystemdToCgroupStyleName(name string) string {
+	return ""
+}
+
+func IsSystemdStyleName(name string) bool {
+	return false
+}

pkg/kubelet/cm/helpers_linux.go

@@ -26,6 +26,7 @@ import (
 	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
 
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/api/v1/resource"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
 	v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
@@ -222,3 +223,8 @@ func getCgroupProcs(dir string) ([]int, error) {
 	}
 	return out, nil
 }
+
+// GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier
+func GetPodCgroupNameSuffix(podUID types.UID) string {
+	return podCgroupNamePrefix + string(podUID)
+}

pkg/kubelet/cm/helpers_unsupported.go

@@ -18,7 +18,10 @@ limitations under the License.
 
 package cm
 
-import "k8s.io/api/core/v1"
+import (
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
 
 const (
 	MinShares     = 0
@@ -52,3 +55,8 @@ func GetCgroupSubsystems() (*CgroupSubsystems, error) {
 func getCgroupProcs(dir string) ([]int, error) {
 	return nil, nil
 }
+
+// GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier
+func GetPodCgroupNameSuffix(podUID types.UID) string {
+	return ""
+}

pkg/kubelet/cm/pod_container_manager_linux.go

@@ -104,7 +104,7 @@ func (m *podContainerManagerImpl) GetPodContainerName(pod *v1.Pod) (CgroupName,
 	case v1.PodQOSBestEffort:
 		parentContainer = m.qosContainersInfo.BestEffort
 	}
-	podContainer := podCgroupNamePrefix + string(pod.UID)
+	podContainer := GetPodCgroupNameSuffix(pod.UID)
 
 	// Get the absolute path of the cgroup
 	cgroupName := (CgroupName)(path.Join(parentContainer, podContainer))

pkg/kubelet/stats/BUILD

@@ -15,6 +15,7 @@ go_library(
         "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
         "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
         "//pkg/kubelet/cadvisor:go_default_library",
+        "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/leaky:go_default_library",
         "//pkg/kubelet/network:go_default_library",

pkg/kubelet/stats/cadvisor_stats_provider.go

@@ -18,6 +18,7 @@ package stats
 
 import (
 	"fmt"
+	"path"
 	"sort"
 	"strings"
 
@@ -28,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
+	"k8s.io/kubernetes/pkg/kubelet/cm"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/leaky"
 	"k8s.io/kubernetes/pkg/kubelet/server/stats"
@@ -89,9 +91,9 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 			return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
 		}
 	}
-
+	// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
+	allInfos := infos
 	infos = removeTerminatedContainerInfo(infos)
-
 	// Map each container to a pod and update the PodStats with container data.
 	podToStats := map[statsapi.PodReference]*statsapi.PodStats{}
 	for key, cinfo := range infos {
@@ -141,6 +143,13 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 			podStats.VolumeStats = append(vstats.EphemeralVolumes, vstats.PersistentVolumes...)
 		}
 		podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo)
+		// Lookup the pod-level cgroup's CPU and memory stats
+		podInfo := getcadvisorPodInfoFromPodUID(podUID, allInfos)
+		if podInfo != nil {
+			cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo)
+			podStats.CPU = cpu
+			podStats.Memory = memory
+		}
 		result = append(result, *podStats)
 	}
 
@@ -243,6 +252,19 @@ func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool {
 	return managed
 }
 
+// getcadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name
+func getcadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo {
+	for key, info := range infos {
+		if cm.IsSystemdStyleName(key) {
+			key = cm.RevertFromSystemdToCgroupStyleName(key)
+		}
+		if cm.GetPodCgroupNameSuffix(podUID) == path.Base(key) {
+			return &info
+		}
+	}
+	return nil
+}
+
 // removeTerminatedContainerInfo returns the specified containerInfo but with
 // the stats of the terminated containers removed.
 //

pkg/kubelet/stats/cadvisor_stats_provider_test.go

@@ -136,6 +136,8 @@ func TestCadvisorListPodStats(t *testing.T) {
 		// Pod2 - Namespace2
 		"/pod2-i":                        getTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName),
 		"/pod2-c0":                       getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20),
+		"/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
+		"/kubepods/podUIDpod1":           getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName),
 	}
 
 	freeRootfsInodes := rootfsInodesFree
@@ -228,6 +230,8 @@ func TestCadvisorListPodStats(t *testing.T) {
 	assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
 	checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)
 	checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, []int{seedEphemeralVolume1, seedEphemeralVolume2}, ps.EphemeralStorage)
+	checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
+	checkMemoryStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Memory)
 
 	// Validate Pod1 Results
 	ps, found = indexPods[prf1]

pkg/kubelet/stats/helper.go

@@ -30,21 +30,15 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/network"
 )
 
-// cadvisorInfoToContainerStats returns the statsapi.ContainerStats converted
+func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsapi.CPUStats, *statsapi.MemoryStats) {
-// from the container and filesystem info.
-func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo, rootFs, imageFs *cadvisorapiv2.FsInfo) *statsapi.ContainerStats {
-	result := &statsapi.ContainerStats{
-		StartTime: metav1.NewTime(info.Spec.CreationTime),
-		Name:      name,
-	}
-
 	cstat, found := latestContainerStats(info)
 	if !found {
-		return result
+		return nil, nil
 	}
-
+	var cpuStats *statsapi.CPUStats
+	var memoryStats *statsapi.MemoryStats
 	if info.Spec.HasCpu {
-		cpuStats := statsapi.CPUStats{
+		cpuStats = &statsapi.CPUStats{
 			Time: metav1.NewTime(cstat.Timestamp),
 		}
 		if cstat.CpuInst != nil {
@@ -53,13 +47,11 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
 		if cstat.Cpu != nil {
 			cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total
 		}
-		result.CPU = &cpuStats
 	}
-
 	if info.Spec.HasMemory {
 		pageFaults := cstat.Memory.ContainerData.Pgfault
 		majorPageFaults := cstat.Memory.ContainerData.Pgmajfault
-		result.Memory = &statsapi.MemoryStats{
+		memoryStats = &statsapi.MemoryStats{
 			Time:            metav1.NewTime(cstat.Timestamp),
 			UsageBytes:      &cstat.Memory.Usage,
 			WorkingSetBytes: &cstat.Memory.WorkingSet,
@@ -70,9 +62,27 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
 		// availableBytes = memory limit (if known) - workingset
 		if !isMemoryUnlimited(info.Spec.Memory.Limit) {
 			availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
-			result.Memory.AvailableBytes = &availableBytes
+			memoryStats.AvailableBytes = &availableBytes
 		}
 	}
+	return cpuStats, memoryStats
+}
+
+// cadvisorInfoToContainerStats returns the statsapi.ContainerStats converted
+// from the container and filesystem info.
+func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo, rootFs, imageFs *cadvisorapiv2.FsInfo) *statsapi.ContainerStats {
+	result := &statsapi.ContainerStats{
+		StartTime: metav1.NewTime(info.Spec.CreationTime),
+		Name:      name,
+	}
+	cstat, found := latestContainerStats(info)
+	if !found {
+		return result
+	}
+
+	cpu, memory := cadvisorInfoToCPUandMemoryStats(info)
+	result.CPU = cpu
+	result.Memory = memory
 
 	if rootFs != nil {
 		// The container logs live on the node rootfs device

test/e2e_node/summary_test.go

@@ -175,6 +175,20 @@ var _ = framework.KubeDescribe("Summary API", func() {
 					"TxBytes":  bounded(10, 10*framework.Mb),
 					"TxErrors": bounded(0, 1000),
 				}),
+				"CPU": ptrMatchAllFields(gstruct.Fields{
+					"Time":                 recent(maxStatsAge),
+					"UsageNanoCores":       bounded(100000, 1E9),
+					"UsageCoreNanoSeconds": bounded(10000000, 1E11),
+				}),
+				"Memory": ptrMatchAllFields(gstruct.Fields{
+					"Time":            recent(maxStatsAge),
+					"AvailableBytes":  bounded(1*framework.Kb, 10*framework.Mb),
+					"UsageBytes":      bounded(10*framework.Kb, 20*framework.Mb),
+					"WorkingSetBytes": bounded(10*framework.Kb, 20*framework.Mb),
+					"RSSBytes":        bounded(1*framework.Kb, framework.Mb),
+					"PageFaults":      bounded(0, 1000000),
+					"MajorPageFaults": bounded(0, 10),
+				}),
 				"VolumeStats": gstruct.MatchAllElements(summaryObjectID, gstruct.Elements{
 					"test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{
 						"Name":   Equal("test-empty-dir"),