github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 12:15:52 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #68841 from krzysztof-jastrzebski/cpuandmemeory2

Browse Source 
Optimizes calculating stats when only CPU and Memory stats are returned from Kubelet stats/summary http endpoint.
This commit is contained in:
k8s-ci-robot 2018-10-18 16:41:07 -07:00 committed by GitHub
commit 4339a70dfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 752 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/kubelet/server/server_test.go
View File

@ -256,13 +256,17 @@ func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Vo
return map[string]volume.Volume{}, true
}
func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error) { return nil, nil }
func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error) { return nil, nil }
func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error) { return nil, nil }
func (_ *fakeKubelet) RlimitStats() (*statsapi.RlimitStats, error) { return nil, nil }
func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error) { return nil, nil }
func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error) { return nil, nil }
func (_ *fakeKubelet) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) { return nil, nil }
func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error) { return nil, nil }
func (_ *fakeKubelet) RlimitStats() (*statsapi.RlimitStats, error) { return nil, nil }
func (_ *fakeKubelet) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
return nil, nil, nil
}
func (_ *fakeKubelet) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error) {
return nil, nil
}
type fakeAuth struct {
authenticateFunc func(*http.Request) (user.Info, bool, error)

5
pkg/kubelet/server/stats/handler.go
View File

@ -43,6 +43,8 @@ type StatsProvider interface {
//
// ListPodStats returns the stats of all the containers managed by pods.
ListPodStats() ([]statsapi.PodStats, error)
// ListPodCPUAndMemoryStats returns the CPU and memory stats of all the containers managed by pods.
ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error)
// ImageFsStats returns the stats of the image filesystem.
ImageFsStats() (*statsapi.FsStats, error)
@ -51,6 +53,9 @@ type StatsProvider interface {
// GetCgroupStats returns the stats and the networking usage of the cgroup
// with the specified cgroupName.
GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error)
// GetCgroupCPUAndMemoryStats returns the CPU and memory stats of the cgroup with the specified cgroupName.
GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error)
// RootFsStats returns the stats of the node root filesystem.
RootFsStats() (*statsapi.FsStats, error)

47
pkg/kubelet/server/stats/summary.go
View File

@ -91,30 +91,33 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error)
}
func (sp *summaryProviderImpl) GetCPUAndMemoryStats() (*statsapi.Summary, error) {
summary, err := sp.Get(false)
// TODO(timstclair): Consider returning a best-effort response if any of
// the following errors occur.
node, err := sp.provider.GetNode()
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to get node info: %v", err)
}
summary.Node.Network = nil
summary.Node.Fs = nil
summary.Node.Runtime = nil
summary.Node.Rlimit = nil
for i := 0; i < len(summary.Node.SystemContainers); i++ {
summary.Node.SystemContainers[i].Accelerators = nil
summary.Node.SystemContainers[i].Rootfs = nil
summary.Node.SystemContainers[i].Logs = nil
summary.Node.SystemContainers[i].UserDefinedMetrics = nil
nodeConfig := sp.provider.GetNodeConfig()
rootStats, err := sp.provider.GetCgroupCPUAndMemoryStats("/", false)
if err != nil {
return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
}
for i := 0; i < len(summary.Pods); i++ {
summary.Pods[i].Network = nil
summary.Pods[i].VolumeStats = nil
summary.Pods[i].EphemeralStorage = nil
for j := 0; j < len(summary.Pods[i].Containers); j++ {
summary.Pods[i].Containers[j].Accelerators = nil
summary.Pods[i].Containers[j].Rootfs = nil
summary.Pods[i].Containers[j].Logs = nil
summary.Pods[i].Containers[j].UserDefinedMetrics = nil
}
podStats, err := sp.provider.ListPodCPUAndMemoryStats()
if err != nil {
return nil, fmt.Errorf("failed to list pod stats: %v", err)
}
return summary, nil
nodeStats := statsapi.NodeStats{
NodeName: node.Name,
CPU: rootStats.CPU,
Memory: rootStats.Memory,
StartTime: rootStats.StartTime,
SystemContainers: sp.GetSystemContainersCPUAndMemoryStats(nodeConfig, podStats, false),
}
summary := statsapi.Summary{
Node: nodeStats,
Pods: podStats,
}
return &summary, nil
}

27
pkg/kubelet/server/stats/summary_sys_containers.go
View File

@ -53,3 +53,30 @@ func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig
return stats
}
func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) {
systemContainers := map[string]struct {
name string
forceStatsUpdate bool
}{
statsapi.SystemContainerKubelet: {nodeConfig.KubeletCgroupsName, false},
statsapi.SystemContainerRuntime: {nodeConfig.RuntimeCgroupsName, false},
statsapi.SystemContainerMisc: {nodeConfig.SystemCgroupsName, false},
statsapi.SystemContainerPods: {sp.provider.GetPodCgroupRoot(), updateStats},
}
for sys, cont := range systemContainers {
// skip if cgroup name is undefined (not all system containers are required)
if cont.name == "" {
continue
}
s, err := sp.provider.GetCgroupCPUAndMemoryStats(cont.name, cont.forceStatsUpdate)
if err != nil {
glog.Errorf("Failed to get system container stats for %q: %v", cont.name, err)
continue
}
s.Name = sys
stats = append(stats, *s)
}
return stats
}

9
pkg/kubelet/server/stats/summary_sys_containers_windows.go
View File

@ -27,11 +27,16 @@ import (
)
func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) {
stats = append(stats, sp.getSystemPodsStats(nodeConfig, podStats, updateStats))
stats = append(stats, sp.getSystemPodsCPUAndMemoryStats(nodeConfig, podStats, updateStats))
return stats
}
func (sp *summaryProviderImpl) getSystemPodsStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) statsapi.ContainerStats {
func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) {
stats = append(stats, sp.getSystemPodsCPUAndMemoryStats(nodeConfig, podStats, updateStats))
return stats
}
func (sp *summaryProviderImpl) getSystemPodsCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) statsapi.ContainerStats {
now := metav1.NewTime(time.Now())
podsSummary := statsapi.ContainerStats{
StartTime: now,

74
pkg/kubelet/server/stats/summary_test.go
View File

@ -33,15 +33,6 @@ import (
)
var (
podStats = []statsapi.PodStats{
{
PodRef: statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"},
StartTime: metav1.NewTime(time.Now()),
Containers: []statsapi.ContainerStats{*getContainerStats()},
Network: getNetworkStats(),
VolumeStats: []statsapi.VolumeStats{*getVolumeStats()},
},
}
imageFsStats = getFsStats()
rootFsStats = getFsStats()
node = &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "test-node"}}
@ -50,8 +41,23 @@ var (
SystemCgroupsName: "/misc",
KubeletCgroupsName: "/kubelet",
}
cgroupRoot = "/kubepods"
cgroupStatsMap = map[string]struct {
cgroupRoot = "/kubepods"
rlimitStats = getRlimitStats()
)
func TestSummaryProviderGetStats(t *testing.T) {
assert := assert.New(t)
podStats := []statsapi.PodStats{
{
PodRef: statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"},
StartTime: metav1.NewTime(time.Now()),
Containers: []statsapi.ContainerStats{*getContainerStats()},
Network: getNetworkStats(),
VolumeStats: []statsapi.VolumeStats{*getVolumeStats()},
},
}
cgroupStatsMap := map[string]struct {
cs *statsapi.ContainerStats
ns *statsapi.NetworkStats
}{
@ -61,11 +67,6 @@ var (
"/kubelet": {cs: getContainerStats(), ns: getNetworkStats()},
"/pods": {cs: getContainerStats(), ns: getNetworkStats()},
}
rlimitStats = getRlimitStats()
)
func TestSummaryProviderGetStats(t *testing.T) {
assert := assert.New(t)
mockStatsProvider := new(statstest.StatsProvider)
mockStatsProvider.
@ -133,20 +134,34 @@ func TestSummaryProviderGetStats(t *testing.T) {
func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) {
assert := assert.New(t)
podStats := []statsapi.PodStats{
{
PodRef: statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"},
StartTime: metav1.NewTime(time.Now()),
Containers: []statsapi.ContainerStats{*getContainerStats()},
},
}
cgroupStatsMap := map[string]struct {
cs *statsapi.ContainerStats
}{
"/": {cs: getVolumeCPUAndMemoryStats()},
"/runtime": {cs: getVolumeCPUAndMemoryStats()},
"/misc": {cs: getVolumeCPUAndMemoryStats()},
"/kubelet": {cs: getVolumeCPUAndMemoryStats()},
"/pods": {cs: getVolumeCPUAndMemoryStats()},
}
mockStatsProvider := new(statstest.StatsProvider)
mockStatsProvider.
On("GetNode").Return(node, nil).
On("GetNodeConfig").Return(nodeConfig).
On("GetPodCgroupRoot").Return(cgroupRoot).
On("ListPodStats").Return(podStats, nil).
On("ImageFsStats").Return(imageFsStats, nil).
On("RootFsStats").Return(rootFsStats, nil).
On("RlimitStats").Return(rlimitStats, nil).
On("GetCgroupStats", "/", false).Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil).
On("GetCgroupStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil).
On("GetCgroupStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil).
On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil).
On("GetCgroupStats", "/kubepods", false).Return(cgroupStatsMap["/pods"].cs, cgroupStatsMap["/pods"].ns, nil)
On("ListPodCPUAndMemoryStats").Return(podStats, nil).
On("GetCgroupCPUAndMemoryStats", "/", false).Return(cgroupStatsMap["/"].cs, nil).
On("GetCgroupCPUAndMemoryStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, nil).
On("GetCgroupCPUAndMemoryStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, nil).
On("GetCgroupCPUAndMemoryStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, nil).
On("GetCgroupCPUAndMemoryStats", "/kubepods", false).Return(cgroupStatsMap["/pods"].cs, nil)
provider := NewSummaryProvider(mockStatsProvider)
summary, err := provider.GetCPUAndMemoryStats()
@ -201,6 +216,15 @@ func getContainerStats() *statsapi.ContainerStats {
f.Fuzz(v)
return v
}
func getVolumeCPUAndMemoryStats() *statsapi.ContainerStats {
f := fuzz.New().NilChance(0)
v := &statsapi.ContainerStats{}
f.Fuzz(&v.Name)
f.Fuzz(&v.StartTime)
f.Fuzz(v.CPU)
f.Fuzz(v.Memory)
return v
}
func getVolumeStats() *statsapi.VolumeStats {
f := fuzz.New().NilChance(0)

46
pkg/kubelet/server/stats/testing/mock_stats_provider.go
View File

@ -64,6 +64,29 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*v
return r0, r1, r2
}
// GetCgroupCPUAndMemoryStats provides a mock function with given fields: cgroupName, updateStats
func (_m *StatsProvider) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*v1alpha1.ContainerStats, error) {
ret := _m.Called(cgroupName, updateStats)
var r0 *v1alpha1.ContainerStats
if rf, ok := ret.Get(0).(func(string, bool) *v1alpha1.ContainerStats); ok {
r0 = rf(cgroupName, updateStats)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*v1alpha1.ContainerStats)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, bool) error); ok {
r1 = rf(cgroupName, updateStats)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetPodByCgroupfs provides the pod that maps to the specified cgroup, as well
// as whether the pod was found.
func (_m *StatsProvider) GetPodByCgroupfs(cgroupfs string) (*corev1.Pod, bool) {
@ -252,6 +275,29 @@ func (_m *StatsProvider) ListPodStats() ([]v1alpha1.PodStats, error) {
return r0, r1
}
// ListPodCPUAndMemoryStats provides a mock function with given fields:
func (_m *StatsProvider) ListPodCPUAndMemoryStats() ([]v1alpha1.PodStats, error) {
ret := _m.Called()
var r0 []v1alpha1.PodStats
if rf, ok := ret.Get(0).(func() []v1alpha1.PodStats); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]v1alpha1.PodStats)
}
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListVolumesForPod provides a mock function with given fields: podUID
func (_m *StatsProvider) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
ret := _m.Called(podUID)

62
pkg/kubelet/stats/cadvisor_stats_provider.go
View File

@ -145,6 +145,68 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
return result, nil
}
// ListPodCPUAndMemoryStats returns the cpu and memory stats of all the pod-managed containers.
func (p *cadvisorStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) {
infos, err := getCadvisorContainerInfo(p.cadvisor)
if err != nil {
return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
}
// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
allInfos := infos
infos = removeTerminatedContainerInfo(infos)
// Map each container to a pod and update the PodStats with container data.
podToStats := map[statsapi.PodReference]*statsapi.PodStats{}
for key, cinfo := range infos {
// On systemd using devicemapper each mount into the container has an
// associated cgroup. We ignore them to ensure we do not get duplicate
// entries in our summary. For details on .mount units:
// http://man7.org/linux/man-pages/man5/systemd.mount.5.html
if strings.HasSuffix(key, ".mount") {
continue
}
// Build the Pod key if this container is managed by a Pod
if !isPodManagedContainer(&cinfo) {
continue
}
ref := buildPodRef(cinfo.Spec.Labels)
// Lookup the PodStats for the pod using the PodRef. If none exists,
// initialize a new entry.
podStats, found := podToStats[ref]
if !found {
podStats = &statsapi.PodStats{PodRef: ref}
podToStats[ref] = podStats
}
// Update the PodStats entry with the stats from the container by
// adding it to podStats.Containers.
containerName := kubetypes.GetContainerName(cinfo.Spec.Labels)
if containerName == leaky.PodInfraContainerName {
// Special case for infrastructure container which is hidden from
// the user and has network stats.
podStats.StartTime = metav1.NewTime(cinfo.Spec.CreationTime)
} else {
podStats.Containers = append(podStats.Containers, *cadvisorInfoToContainerCPUAndMemoryStats(containerName, &cinfo))
}
}
// Add each PodStats to the result.
result := make([]statsapi.PodStats, 0, len(podToStats))
for _, podStats := range podToStats {
podUID := types.UID(podStats.PodRef.UID)
// Lookup the pod-level cgroup's CPU and memory stats
podInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos)
if podInfo != nil {
cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo)
podStats.CPU = cpu
podStats.Memory = memory
}
result = append(result, *podStats)
}
return result, nil
}
func calcEphemeralStorage(containers []statsapi.ContainerStats, volumes []statsapi.VolumeStats, rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.FsStats {
result := &statsapi.FsStats{
Time: metav1.NewTime(rootFsInfo.Timestamp),

163
pkg/kubelet/stats/cadvisor_stats_provider_test.go
View File

@ -258,6 +258,169 @@ func TestCadvisorListPodStats(t *testing.T) {
checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
}
func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
const (
namespace0 = "test0"
namespace2 = "test2"
)
const (
seedRoot = 0
seedRuntime = 100
seedKubelet = 200
seedMisc = 300
seedPod0Infra = 1000
seedPod0Container0 = 2000
seedPod0Container1 = 2001
seedPod1Infra = 3000
seedPod1Container = 4000
seedPod2Infra = 5000
seedPod2Container = 6000
seedEphemeralVolume1 = 10000
seedEphemeralVolume2 = 10001
seedPersistentVolume1 = 20000
seedPersistentVolume2 = 20001
)
const (
pName0 = "pod0"
pName1 = "pod1"
pName2 = "pod0" // ensure pName2 conflicts with pName0, but is in a different namespace
)
const (
cName00 = "c0"
cName01 = "c1"
cName10 = "c0" // ensure cName10 conflicts with cName02, but is in a different pod
cName20 = "c1" // ensure cName20 conflicts with cName01, but is in a different pod + namespace
)
prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0}
prf1 := statsapi.PodReference{Name: pName1, Namespace: namespace0, UID: "UID" + pName1}
prf2 := statsapi.PodReference{Name: pName2, Namespace: namespace2, UID: "UID" + pName2}
infos := map[string]cadvisorapiv2.ContainerInfo{
"/": getTestContainerInfo(seedRoot, "", "", ""),
"/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""),
"/kubelet": getTestContainerInfo(seedKubelet, "", "", ""),
"/system": getTestContainerInfo(seedMisc, "", "", ""),
// Pod0 - Namespace0
"/pod0-i": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00),
"/pod0-c1": getTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01),
// Pod1 - Namespace0
"/pod1-i": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName),
"/pod1-c0": getTestContainerInfo(seedPod1Container, pName1, namespace0, cName10),
// Pod2 - Namespace2
"/pod2-i": getTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName),
"/pod2-c0": getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20),
"/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
"/kubepods/podUIDpod1": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName),
}
// memory limit overrides for each container (used to test available bytes if a memory limit is known)
memoryLimitOverrides := map[string]uint64{
"/": uint64(1 << 30),
"/pod2-c0": uint64(1 << 15),
}
for name, memoryLimitOverride := range memoryLimitOverrides {
info, found := infos[name]
if !found {
t.Errorf("No container defined with name %v", name)
}
info.Spec.Memory.Limit = memoryLimitOverride
infos[name] = info
}
options := cadvisorapiv2.RequestOptions{
IdType: cadvisorapiv2.TypeName,
Count: 2,
Recursive: true,
}
mockCadvisor := new(cadvisortest.Mock)
mockCadvisor.
On("ContainerInfoV2", "/", options).Return(infos, nil)
ephemeralVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedEphemeralVolume1, "ephemeralVolume1"),
getPodVolumeStats(seedEphemeralVolume2, "ephemeralVolume2")}
persistentVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedPersistentVolume1, "persistentVolume1"),
getPodVolumeStats(seedPersistentVolume2, "persistentVolume2")}
volumeStats := serverstats.PodVolumeStats{
EphemeralVolumes: ephemeralVolumes,
PersistentVolumes: persistentVolumes,
}
resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, nil)
pods, err := p.ListPodCPUAndMemoryStats()
assert.NoError(t, err)
assert.Equal(t, 3, len(pods))
indexPods := make(map[statsapi.PodReference]statsapi.PodStats, len(pods))
for _, pod := range pods {
indexPods[pod.PodRef] = pod
}
// Validate Pod0 Results
ps, found := indexPods[prf0]
assert.True(t, found)
assert.Len(t, ps.Containers, 2)
indexCon := make(map[string]statsapi.ContainerStats, len(ps.Containers))
for _, con := range ps.Containers {
indexCon[con.Name] = con
}
con := indexCon[cName00]
assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix())
checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU)
checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory)
assert.Nil(t, con.Rootfs)
assert.Nil(t, con.Logs)
assert.Nil(t, con.Accelerators)
assert.Nil(t, con.UserDefinedMetrics)
con = indexCon[cName01]
assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU)
checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory)
assert.Nil(t, con.Rootfs)
assert.Nil(t, con.Logs)
assert.Nil(t, con.Accelerators)
assert.Nil(t, con.UserDefinedMetrics)
assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
assert.Nil(t, ps.EphemeralStorage)
assert.Nil(t, ps.VolumeStats)
assert.Nil(t, ps.Network)
if ps.CPU != nil {
checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
}
if ps.Memory != nil {
checkMemoryStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Memory)
}
// Validate Pod1 Results
ps, found = indexPods[prf1]
assert.True(t, found)
assert.Len(t, ps.Containers, 1)
con = ps.Containers[0]
assert.Equal(t, cName10, con.Name)
checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU)
checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory)
assert.Nil(t, ps.EphemeralStorage)
assert.Nil(t, ps.VolumeStats)
assert.Nil(t, ps.Network)
// Validate Pod2 Results
ps, found = indexPods[prf2]
assert.True(t, found)
assert.Len(t, ps.Containers, 1)
con = ps.Containers[0]
assert.Equal(t, cName20, con.Name)
checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU)
checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory)
assert.Nil(t, ps.EphemeralStorage)
assert.Nil(t, ps.VolumeStats)
assert.Nil(t, ps.Network)
}
func TestCadvisorImagesFsStats(t *testing.T) {
var (
assert = assert.New(t)

108
pkg/kubelet/stats/cri_stats_provider.go
View File

@ -169,6 +169,87 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
return result, nil
}
// ListPodCPUAndMemoryStats returns the CPU and Memory stats of all the pod-managed containers.
func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) {
containers, err := p.runtimeService.ListContainers(&runtimeapi.ContainerFilter{})
if err != nil {
return nil, fmt.Errorf("failed to list all containers: %v", err)
}
// Creates pod sandbox map.
podSandboxMap := make(map[string]*runtimeapi.PodSandbox)
podSandboxes, err := p.runtimeService.ListPodSandbox(&runtimeapi.PodSandboxFilter{})
if err != nil {
return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err)
}
for _, s := range podSandboxes {
podSandboxMap[s.Id] = s
}
// sandboxIDToPodStats is a temporary map from sandbox ID to its pod stats.
sandboxIDToPodStats := make(map[string]*statsapi.PodStats)
resp, err := p.runtimeService.ListContainerStats(&runtimeapi.ContainerStatsFilter{})
if err != nil {
return nil, fmt.Errorf("failed to list all container stats: %v", err)
}
containers = removeTerminatedContainer(containers)
// Creates container map.
containerMap := make(map[string]*runtimeapi.Container)
for _, c := range containers {
containerMap[c.Id] = c
}
allInfos, err := getCadvisorContainerInfo(p.cadvisor)
if err != nil {
return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
}
caInfos := getCRICadvisorStats(allInfos)
for _, stats := range resp {
containerID := stats.Attributes.Id
container, found := containerMap[containerID]
if !found {
continue
}
podSandboxID := container.PodSandboxId
podSandbox, found := podSandboxMap[podSandboxID]
if !found {
continue
}
// Creates the stats of the pod (if not created yet) which the
// container belongs to.
ps, found := sandboxIDToPodStats[podSandboxID]
if !found {
ps = buildPodStats(podSandbox)
sandboxIDToPodStats[podSandboxID] = ps
}
// Fill available CPU and memory stats for full set of required pod stats
cs := p.makeContainerCPUAndMemoryStats(stats, container)
p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
// If cadvisor stats is available for the container, use it to populate
// container stats
caStats, caFound := caInfos[containerID]
if !caFound {
glog.V(4).Infof("Unable to find cadvisor stats for %q", containerID)
} else {
p.addCadvisorContainerStats(cs, &caStats)
}
ps.Containers = append(ps.Containers, *cs)
}
result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats))
for _, s := range sandboxIDToPodStats {
result = append(result, *s)
}
return result, nil
}
// ImageFsStats returns the stats of the image filesystem.
func (p *criStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
resp, err := p.imageService.ImageFsInfo()
@ -393,6 +474,33 @@ func (p *criStatsProvider) makeContainerStats(
return result
}
func (p *criStatsProvider) makeContainerCPUAndMemoryStats(
stats *runtimeapi.ContainerStats,
container *runtimeapi.Container,
) *statsapi.ContainerStats {
result := &statsapi.ContainerStats{
Name: stats.Attributes.Metadata.Name,
// The StartTime in the summary API is the container creation time.
StartTime: metav1.NewTime(time.Unix(0, container.CreatedAt)),
CPU: &statsapi.CPUStats{},
Memory: &statsapi.MemoryStats{},
// UserDefinedMetrics is not supported by CRI.
}
if stats.Cpu != nil {
result.CPU.Time = metav1.NewTime(time.Unix(0, stats.Cpu.Timestamp))
if stats.Cpu.UsageCoreNanoSeconds != nil {
result.CPU.UsageCoreNanoSeconds = &stats.Cpu.UsageCoreNanoSeconds.Value
}
}
if stats.Memory != nil {
result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp))
if stats.Memory.WorkingSetBytes != nil {
result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value
}
}
return result
}
// removeTerminatedContainer returns the specified container but with
// the stats of the terminated containers removed.
func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi.Container {

212
pkg/kubelet/stats/cri_stats_provider_test.go
View File

@ -46,33 +46,33 @@ const (
offsetUsage
)
const (
seedRoot = 0
seedKubelet = 200
seedMisc = 300
seedSandbox0 = 1000
seedContainer0 = 2000
seedSandbox1 = 3000
seedContainer1 = 4000
seedContainer2 = 5000
seedSandbox2 = 6000
seedContainer3 = 7000
)
const (
pName0 = "pod0"
pName1 = "pod1"
pName2 = "pod2"
)
const (
cName0 = "container0-name"
cName1 = "container1-name"
cName2 = "container2-name"
cName3 = "container3-name"
)
func TestCRIListPodStats(t *testing.T) {
const (
seedRoot = 0
seedKubelet = 200
seedMisc = 300
seedSandbox0 = 1000
seedContainer0 = 2000
seedSandbox1 = 3000
seedContainer1 = 4000
seedContainer2 = 5000
seedSandbox2 = 6000
seedContainer3 = 7000
)
const (
pName0 = "pod0"
pName1 = "pod1"
pName2 = "pod2"
)
const (
cName0 = "container0-name"
cName1 = "container1-name"
cName2 = "container2-name"
cName3 = "container3-name"
)
var (
imageFsMountpoint = "/test/mount/point"
unknownMountpoint = "/unknown/mount/point"
@ -242,6 +242,166 @@ func TestCRIListPodStats(t *testing.T) {
mockCadvisor.AssertExpectations(t)
}
func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
var (
imageFsMountpoint = "/test/mount/point"
unknownMountpoint = "/unknown/mount/point"
sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
sandbox0Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox0.PodSandboxStatus.Metadata.Uid))
container0 = makeFakeContainer(sandbox0, cName0, 0, false)
containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint)
container1 = makeFakeContainer(sandbox0, cName1, 0, false)
containerStats1 = makeFakeContainerStats(container1, unknownMountpoint)
sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
sandbox1Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox1.PodSandboxStatus.Metadata.Uid))
container2 = makeFakeContainer(sandbox1, cName2, 0, false)
containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint)
sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
sandbox2Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox2.PodSandboxStatus.Metadata.Uid))
container3 = makeFakeContainer(sandbox2, cName3, 0, true)
containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint)
container4 = makeFakeContainer(sandbox2, cName3, 1, false)
containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
)
var (
mockCadvisor = new(cadvisortest.Mock)
mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
mockPodManager = new(kubepodtest.MockManager)
resourceAnalyzer = new(fakeResourceAnalyzer)
fakeRuntimeService = critest.NewFakeRuntimeService()
)
infos := map[string]cadvisorapiv2.ContainerInfo{
"/": getTestContainerInfo(seedRoot, "", "", ""),
"/kubelet": getTestContainerInfo(seedKubelet, "", "", ""),
"/system": getTestContainerInfo(seedMisc, "", "", ""),
sandbox0.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
sandbox0Cgroup: getTestContainerInfo(seedSandbox0, "", "", ""),
container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0),
container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1),
sandbox1.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox1, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
sandbox1Cgroup: getTestContainerInfo(seedSandbox1, "", "", ""),
container2.ContainerStatus.Id: getTestContainerInfo(seedContainer2, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, cName2),
sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
sandbox2Cgroup: getTestContainerInfo(seedSandbox2, "", "", ""),
container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
}
options := cadvisorapiv2.RequestOptions{
IdType: cadvisorapiv2.TypeName,
Count: 2,
Recursive: true,
}
mockCadvisor.
On("ContainerInfoV2", "/", options).Return(infos, nil)
fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{
sandbox0, sandbox1, sandbox2,
})
fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
container0, container1, container2, container3, container4,
})
fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
containerStats0, containerStats1, containerStats2, containerStats3, containerStats4,
})
ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"})
persistentVolumes := makeFakeVolumeStats([]string{"persisVolume1, persisVolumes2"})
resourceAnalyzer.podVolumeStats = serverstats.PodVolumeStats{
EphemeralVolumes: ephemeralVolumes,
PersistentVolumes: persistentVolumes,
}
provider := NewCRIStatsProvider(
mockCadvisor,
resourceAnalyzer,
mockPodManager,
mockRuntimeCache,
fakeRuntimeService,
nil,
nil,
)
stats, err := provider.ListPodCPUAndMemoryStats()
assert := assert.New(t)
assert.NoError(err)
assert.Equal(3, len(stats))
podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats)
for _, s := range stats {
podStatsMap[s.PodRef] = s
}
p0 := podStatsMap[statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}]
assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
assert.Equal(2, len(p0.Containers))
assert.Nil(p0.EphemeralStorage)
assert.Nil(p0.VolumeStats)
assert.Nil(p0.Network)
checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
containerStatsMap := make(map[string]statsapi.ContainerStats)
for _, s := range p0.Containers {
containerStatsMap[s.Name] = s
}
c0 := containerStatsMap[cName0]
assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
assert.Nil(c0.Rootfs)
assert.Nil(c0.Logs)
assert.Nil(c0.Accelerators)
assert.Nil(c0.UserDefinedMetrics)
c1 := containerStatsMap[cName1]
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
assert.Nil(c1.Rootfs)
assert.Nil(c1.Logs)
assert.Nil(c1.Accelerators)
assert.Nil(c1.UserDefinedMetrics)
p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
assert.Equal(1, len(p1.Containers))
assert.Nil(p1.EphemeralStorage)
assert.Nil(p1.VolumeStats)
assert.Nil(p1.Network)
checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
c2 := p1.Containers[0]
assert.Equal(cName2, c2.Name)
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
assert.Nil(c2.Rootfs)
assert.Nil(c2.Logs)
assert.Nil(c2.Accelerators)
assert.Nil(c2.UserDefinedMetrics)
p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
assert.Equal(1, len(p2.Containers))
assert.Nil(p2.EphemeralStorage)
assert.Nil(p2.VolumeStats)
assert.Nil(p2.Network)
checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
c3 := p2.Containers[0]
assert.Equal(cName3, c3.Name)
assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
assert.Nil(c2.Rootfs)
assert.Nil(c2.Logs)
assert.Nil(c2.Accelerators)
assert.Nil(c2.UserDefinedMetrics)
mockCadvisor.AssertExpectations(t)
}
func TestCRIImagesFsStats(t *testing.T) {
var (
imageFsMountpoint = "/test/mount/point"

15
pkg/kubelet/stats/helper.go
View File

@ -132,6 +132,21 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
return result
}
// cadvisorInfoToContainerCPUAndMemoryStats returns the statsapi.ContainerStats converted
// from the container and filesystem info.
func cadvisorInfoToContainerCPUAndMemoryStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.ContainerStats {
result := &statsapi.ContainerStats{
StartTime: metav1.NewTime(info.Spec.CreationTime),
Name: name,
}
cpu, memory := cadvisorInfoToCPUandMemoryStats(info)
result.CPU = cpu
result.Memory = memory
return result
}
// cadvisorInfoToNetworkStats returns the statsapi.NetworkStats converted from
// the container info from cadvisor.
func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.NetworkStats {

13
pkg/kubelet/stats/stats_provider.go
View File

@ -85,6 +85,7 @@ type StatsProvider struct {
// containers managed by pods.
type containerStatsProvider interface {
ListPodStats() ([]statsapi.PodStats, error)
ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error)
ImageFsStats() (*statsapi.FsStats, error)
ImageFsDevice() (string, error)
}
@ -106,6 +107,18 @@ func (p *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*st
return s, n, nil
}
// GetCgroupCPUAndMemoryStats returns the CPU and memory stats of the cgroup with the cgroupName. Note that
// this function doesn't generate filesystem stats.
func (p *StatsProvider) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error) {
info, err := getCgroupInfo(p.cadvisor, cgroupName, updateStats)
if err != nil {
return nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err)
}
// Rootfs and imagefs doesn't make sense for raw cgroup.
s := cadvisorInfoToContainerCPUAndMemoryStats(cgroupName, info)
return s, nil
}
// RootFsStats returns the stats of the node root filesystem.
func (p *StatsProvider) RootFsStats() (*statsapi.FsStats, error) {
rootFsInfo, err := p.cadvisor.RootFsInfo()

38
pkg/kubelet/stats/stats_provider_test.go
View File

@ -100,6 +100,39 @@ func TestGetCgroupStats(t *testing.T) {
mockCadvisor.AssertExpectations(t)
}
func TestGetCgroupCPUAndMemoryStats(t *testing.T) {
const (
cgroupName = "test-cgroup-name"
containerInfoSeed = 1000
updateStats = false
)
var (
mockCadvisor = new(cadvisortest.Mock)
mockPodManager = new(kubepodtest.MockManager)
mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
assert = assert.New(t)
options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false}
containerInfo = getTestContainerInfo(containerInfoSeed, "test-pod", "test-ns", "test-container")
containerInfoMap = map[string]cadvisorapiv2.ContainerInfo{cgroupName: containerInfo}
)
mockCadvisor.On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil)
provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{})
cs, err := provider.GetCgroupCPUAndMemoryStats(cgroupName, updateStats)
assert.NoError(err)
checkCPUStats(t, "", containerInfoSeed, cs.CPU)
checkMemoryStats(t, "", containerInfoSeed, containerInfo, cs.Memory)
assert.Equal(cgroupName, cs.Name)
assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime)
mockCadvisor.AssertExpectations(t)
}
func TestRootFsStats(t *testing.T) {
const (
rootFsInfoSeed = 1000
@ -648,6 +681,11 @@ type fakeContainerStatsProvider struct {
func (p fakeContainerStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
return nil, fmt.Errorf("not implemented")
}
func (p fakeContainerStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) {
return nil, fmt.Errorf("not implemented")
}
func (p fakeContainerStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
return nil, fmt.Errorf("not implemented")
}