diff --git a/pkg/kubelet/apis/stats/v1alpha1/types.go b/pkg/kubelet/apis/stats/v1alpha1/types.go index 36aac266cde..d0d2707ec0c 100644 --- a/pkg/kubelet/apis/stats/v1alpha1/types.go +++ b/pkg/kubelet/apis/stats/v1alpha1/types.go @@ -86,6 +86,12 @@ type PodStats struct { // +patchMergeKey=name // +patchStrategy=merge Containers []ContainerStats `json:"containers" patchStrategy:"merge" patchMergeKey:"name"` + // Stats pertaining to CPU resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead). + // +optional + CPU *CPUStats `json:"cpu,omitempty"` + // Stats pertaining to memory (RAM) resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead). + // +optional + Memory *MemoryStats `json:"memory,omitempty"` // Stats pertaining to network resources. // +optional Network *NetworkStats `json:"network,omitempty"` diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go index 62894e5a3dd..d1d99713429 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_manager_linux.go @@ -45,6 +45,8 @@ const ( libcontainerCgroupfs libcontainerCgroupManagerType = "cgroupfs" // libcontainerSystemd means use libcontainer with systemd libcontainerSystemd libcontainerCgroupManagerType = "systemd" + // systemdSuffix is the cgroup name suffix for systemd + systemdSuffix string = ".slice" ) // hugePageSizeList is useful for converting to the hugetlb canonical unit @@ -68,8 +70,8 @@ func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) st } // detect if we are given a systemd style name. // if so, we do not want to do double encoding. - if strings.HasSuffix(part, ".slice") { - part = strings.TrimSuffix(part, ".slice") + if IsSystemdStyleName(part) { + part = strings.TrimSuffix(part, systemdSuffix) separatorIndex := strings.LastIndex(part, "-") if separatorIndex >= 0 && separatorIndex < len(part) { part = part[separatorIndex+1:] @@ -87,8 +89,8 @@ func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) st result = "-" } // always have a .slice suffix - if !strings.HasSuffix(result, ".slice") { - result = result + ".slice" + if !IsSystemdStyleName(result) { + result = result + systemdSuffix } // if the caller desired the result in cgroupfs format... @@ -114,6 +116,13 @@ func ConvertCgroupFsNameToSystemd(cgroupfsName string) (string, error) { return path.Base(cgroupfsName), nil } +func IsSystemdStyleName(name string) bool { + if strings.HasSuffix(name, systemdSuffix) { + return true + } + return false +} + // libcontainerAdapter provides a simplified interface to libcontainer based on libcontainer type. type libcontainerAdapter struct { // cgroupManagerType defines how to interface with libcontainer @@ -151,15 +160,18 @@ func (l *libcontainerAdapter) revertName(name string) CgroupName { if l.cgroupManagerType != libcontainerSystemd { return CgroupName(name) } + return CgroupName(RevertFromSystemdToCgroupStyleName(name)) +} +func RevertFromSystemdToCgroupStyleName(name string) string { driverName, err := ConvertCgroupFsNameToSystemd(name) if err != nil { panic(err) } - driverName = strings.TrimSuffix(driverName, ".slice") + driverName = strings.TrimSuffix(driverName, systemdSuffix) driverName = strings.Replace(driverName, "-", "/", -1) driverName = strings.Replace(driverName, "_", "-", -1) - return CgroupName(driverName) + return driverName } // adaptName converts a CgroupName identifier to a driver specific conversion value. diff --git a/pkg/kubelet/cm/cgroup_manager_unsupported.go b/pkg/kubelet/cm/cgroup_manager_unsupported.go index 6a567e94b15..34345985cf9 100644 --- a/pkg/kubelet/cm/cgroup_manager_unsupported.go +++ b/pkg/kubelet/cm/cgroup_manager_unsupported.go @@ -77,3 +77,11 @@ func ConvertCgroupFsNameToSystemd(cgroupfsName string) (string, error) { func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) string { return "" } + +func RevertFromSystemdToCgroupStyleName(name string) string { + return "" +} + +func IsSystemdStyleName(name string) bool { + return false +} diff --git a/pkg/kubelet/cm/helpers_linux.go b/pkg/kubelet/cm/helpers_linux.go index e36e97be018..935fb6c8060 100644 --- a/pkg/kubelet/cm/helpers_linux.go +++ b/pkg/kubelet/cm/helpers_linux.go @@ -26,6 +26,7 @@ import ( libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/api/v1/resource" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" @@ -222,3 +223,8 @@ func getCgroupProcs(dir string) ([]int, error) { } return out, nil } + +// GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier +func GetPodCgroupNameSuffix(podUID types.UID) string { + return podCgroupNamePrefix + string(podUID) +} diff --git a/pkg/kubelet/cm/helpers_unsupported.go b/pkg/kubelet/cm/helpers_unsupported.go index 3b88c7360c6..b572f3456f3 100644 --- a/pkg/kubelet/cm/helpers_unsupported.go +++ b/pkg/kubelet/cm/helpers_unsupported.go @@ -18,7 +18,10 @@ limitations under the License. package cm -import "k8s.io/api/core/v1" +import ( + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) const ( MinShares = 0 @@ -52,3 +55,8 @@ func GetCgroupSubsystems() (*CgroupSubsystems, error) { func getCgroupProcs(dir string) ([]int, error) { return nil, nil } + +// GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier +func GetPodCgroupNameSuffix(podUID types.UID) string { + return "" +} diff --git a/pkg/kubelet/cm/pod_container_manager_linux.go b/pkg/kubelet/cm/pod_container_manager_linux.go index 2043596c91d..e62d192891d 100644 --- a/pkg/kubelet/cm/pod_container_manager_linux.go +++ b/pkg/kubelet/cm/pod_container_manager_linux.go @@ -104,7 +104,7 @@ func (m *podContainerManagerImpl) GetPodContainerName(pod *v1.Pod) (CgroupName, case v1.PodQOSBestEffort: parentContainer = m.qosContainersInfo.BestEffort } - podContainer := podCgroupNamePrefix + string(pod.UID) + podContainer := GetPodCgroupNameSuffix(pod.UID) // Get the absolute path of the cgroup cgroupName := (CgroupName)(path.Join(parentContainer, podContainer)) diff --git a/pkg/kubelet/stats/BUILD b/pkg/kubelet/stats/BUILD index 5f3af9771c8..07732d9ad09 100644 --- a/pkg/kubelet/stats/BUILD +++ b/pkg/kubelet/stats/BUILD @@ -15,6 +15,7 @@ go_library( "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", + "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/leaky:go_default_library", "//pkg/kubelet/network:go_default_library", diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 6371f0d1c7d..b498c5158ee 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -18,6 +18,7 @@ package stats import ( "fmt" + "path" "sort" "strings" @@ -28,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/leaky" "k8s.io/kubernetes/pkg/kubelet/server/stats" @@ -89,9 +91,9 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { return nil, fmt.Errorf("failed to get root cgroup stats: %v", err) } } - + // removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first + allInfos := infos infos = removeTerminatedContainerInfo(infos) - // Map each container to a pod and update the PodStats with container data. podToStats := map[statsapi.PodReference]*statsapi.PodStats{} for key, cinfo := range infos { @@ -141,6 +143,13 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { podStats.VolumeStats = append(vstats.EphemeralVolumes, vstats.PersistentVolumes...) } podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo) + // Lookup the pod-level cgroup's CPU and memory stats + podInfo := getcadvisorPodInfoFromPodUID(podUID, allInfos) + if podInfo != nil { + cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo) + podStats.CPU = cpu + podStats.Memory = memory + } result = append(result, *podStats) } @@ -243,6 +252,19 @@ func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool { return managed } +// getcadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name +func getcadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo { + for key, info := range infos { + if cm.IsSystemdStyleName(key) { + key = cm.RevertFromSystemdToCgroupStyleName(key) + } + if cm.GetPodCgroupNameSuffix(podUID) == path.Base(key) { + return &info + } + } + return nil +} + // removeTerminatedContainerInfo returns the specified containerInfo but with // the stats of the terminated containers removed. // diff --git a/pkg/kubelet/stats/cadvisor_stats_provider_test.go b/pkg/kubelet/stats/cadvisor_stats_provider_test.go index dee0f11f1bb..2eb9d4a25c9 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider_test.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider_test.go @@ -134,8 +134,10 @@ func TestCadvisorListPodStats(t *testing.T) { "/pod1-i": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName), "/pod1-c0": getTestContainerInfo(seedPod1Container, pName1, namespace0, cName10), // Pod2 - Namespace2 - "/pod2-i": getTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName), - "/pod2-c0": getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20), + "/pod2-i": getTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName), + "/pod2-c0": getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20), + "/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName), + "/kubepods/podUIDpod1": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName), } freeRootfsInodes := rootfsInodesFree @@ -228,6 +230,8 @@ func TestCadvisorListPodStats(t *testing.T) { assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix()) checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network) checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, []int{seedEphemeralVolume1, seedEphemeralVolume2}, ps.EphemeralStorage) + checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU) + checkMemoryStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Memory) // Validate Pod1 Results ps, found = indexPods[prf1] diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index 9b416d7e39f..6315c315f20 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -30,21 +30,15 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" ) -// cadvisorInfoToContainerStats returns the statsapi.ContainerStats converted -// from the container and filesystem info. -func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo, rootFs, imageFs *cadvisorapiv2.FsInfo) *statsapi.ContainerStats { - result := &statsapi.ContainerStats{ - StartTime: metav1.NewTime(info.Spec.CreationTime), - Name: name, - } - +func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsapi.CPUStats, *statsapi.MemoryStats) { cstat, found := latestContainerStats(info) if !found { - return result + return nil, nil } - + var cpuStats *statsapi.CPUStats + var memoryStats *statsapi.MemoryStats if info.Spec.HasCpu { - cpuStats := statsapi.CPUStats{ + cpuStats = &statsapi.CPUStats{ Time: metav1.NewTime(cstat.Timestamp), } if cstat.CpuInst != nil { @@ -53,13 +47,11 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo if cstat.Cpu != nil { cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total } - result.CPU = &cpuStats } - if info.Spec.HasMemory { pageFaults := cstat.Memory.ContainerData.Pgfault majorPageFaults := cstat.Memory.ContainerData.Pgmajfault - result.Memory = &statsapi.MemoryStats{ + memoryStats = &statsapi.MemoryStats{ Time: metav1.NewTime(cstat.Timestamp), UsageBytes: &cstat.Memory.Usage, WorkingSetBytes: &cstat.Memory.WorkingSet, @@ -70,9 +62,27 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo // availableBytes = memory limit (if known) - workingset if !isMemoryUnlimited(info.Spec.Memory.Limit) { availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet - result.Memory.AvailableBytes = &availableBytes + memoryStats.AvailableBytes = &availableBytes } } + return cpuStats, memoryStats +} + +// cadvisorInfoToContainerStats returns the statsapi.ContainerStats converted +// from the container and filesystem info. +func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo, rootFs, imageFs *cadvisorapiv2.FsInfo) *statsapi.ContainerStats { + result := &statsapi.ContainerStats{ + StartTime: metav1.NewTime(info.Spec.CreationTime), + Name: name, + } + cstat, found := latestContainerStats(info) + if !found { + return result + } + + cpu, memory := cadvisorInfoToCPUandMemoryStats(info) + result.CPU = cpu + result.Memory = memory if rootFs != nil { // The container logs live on the node rootfs device diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go index a0c3188ea8c..b308d62f7b7 100644 --- a/test/e2e_node/summary_test.go +++ b/test/e2e_node/summary_test.go @@ -175,6 +175,20 @@ var _ = framework.KubeDescribe("Summary API", func() { "TxBytes": bounded(10, 10*framework.Mb), "TxErrors": bounded(0, 1000), }), + "CPU": ptrMatchAllFields(gstruct.Fields{ + "Time": recent(maxStatsAge), + "UsageNanoCores": bounded(100000, 1E9), + "UsageCoreNanoSeconds": bounded(10000000, 1E11), + }), + "Memory": ptrMatchAllFields(gstruct.Fields{ + "Time": recent(maxStatsAge), + "AvailableBytes": bounded(1*framework.Kb, 10*framework.Mb), + "UsageBytes": bounded(10*framework.Kb, 20*framework.Mb), + "WorkingSetBytes": bounded(10*framework.Kb, 20*framework.Mb), + "RSSBytes": bounded(1*framework.Kb, framework.Mb), + "PageFaults": bounded(0, 1000000), + "MajorPageFaults": bounded(0, 10), + }), "VolumeStats": gstruct.MatchAllElements(summaryObjectID, gstruct.Elements{ "test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{ "Name": Equal("test-empty-dir"),