github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #88734 from joelsmith/master

Browse Source 
Work-around for missing memory metrics on CRI-O exited containers
This commit is contained in:
Kubernetes Prow Robot 2020-03-10 16:21:36 -07:00 committed by GitHub
commit 7989ca4324
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 82 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
pkg/kubelet/stats/cadvisor_stats_provider_test.go
View File

@ -89,6 +89,9 @@ func TestCadvisorListPodStats(t *testing.T) {
seedPod1Container = 4000
seedPod2Infra = 5000
seedPod2Container = 6000
seedPod3Infra = 7000
seedPod3Container0 = 8000
seedPod3Container1 = 8001
seedEphemeralVolume1 = 10000
seedEphemeralVolume2 = 10001
seedPersistentVolume1 = 20000
@ -98,12 +101,15 @@ func TestCadvisorListPodStats(t *testing.T) {
pName0 = "pod0"
pName1 = "pod1"
pName2 = "pod0" // ensure pName2 conflicts with pName0, but is in a different namespace
pName3 = "pod3"
)
const (
cName00 = "c0"
cName01 = "c1"
cName10 = "c0" // ensure cName10 conflicts with cName02, but is in a different pod
cName20 = "c1" // ensure cName20 conflicts with cName01, but is in a different pod + namespace
cName30 = "c0-init"
cName31 = "c1"
)
const (
rootfsCapacity = uint64(10000000)
@ -119,6 +125,7 @@ func TestCadvisorListPodStats(t *testing.T) {
prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0}
prf1 := statsapi.PodReference{Name: pName1, Namespace: namespace0, UID: "UID" + pName1}
prf2 := statsapi.PodReference{Name: pName2, Namespace: namespace2, UID: "UID" + pName2}
prf3 := statsapi.PodReference{Name: pName3, Namespace: namespace0, UID: "UID" + pName3}
infos := map[string]cadvisorapiv2.ContainerInfo{
"/": getTestContainerInfo(seedRoot, "", "", ""),
"/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""),
@ -136,6 +143,10 @@ func TestCadvisorListPodStats(t *testing.T) {
"/pod2-c0": getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20),
"/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
"/kubepods/podUIDpod1": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName),
// Pod3 - Namespace0
"/pod3-i": getTestContainerInfo(seedPod3Infra, pName3, namespace0, leaky.PodInfraContainerName),
"/pod3-c0-init": getTestContainerInfo(seedPod3Container0, pName3, namespace0, cName30),
"/pod3-c1": getTestContainerInfo(seedPod3Container1, pName3, namespace0, cName31),
}
freeRootfsInodes := rootfsInodesFree
@ -169,6 +180,21 @@ func TestCadvisorListPodStats(t *testing.T) {
info.Spec.Memory.Limit = memoryLimitOverride
infos[name] = info
}
// any container for which cadvisor should return no stats (as might be the case for an exited init container)
nostatsOverrides := []string{
"/pod3-c0-init",
}
for _, name := range nostatsOverrides {
info, found := infos[name]
if !found {
t.Errorf("No container defined with name %v", name)
}
info.Spec.Memory = cadvisorapiv2.MemorySpec{}
info.Spec.Cpu = cadvisorapiv2.CpuSpec{}
info.Spec.HasMemory = false
info.Spec.HasCpu = false
infos[name] = info
}
options := cadvisorapiv2.RequestOptions{
IdType: cadvisorapiv2.TypeName,
@ -197,10 +223,12 @@ func TestCadvisorListPodStats(t *testing.T) {
p0Time := metav1.Now()
p1Time := metav1.Now()
p2Time := metav1.Now()
p3Time := metav1.Now()
mockStatus := new(statustest.MockStatusProvider)
mockStatus.On("GetPodStatus", types.UID("UID"+pName0)).Return(v1.PodStatus{StartTime: &p0Time}, true)
mockStatus.On("GetPodStatus", types.UID("UID"+pName1)).Return(v1.PodStatus{StartTime: &p1Time}, true)
mockStatus.On("GetPodStatus", types.UID("UID"+pName2)).Return(v1.PodStatus{StartTime: &p2Time}, true)
mockStatus.On("GetPodStatus", types.UID("UID"+pName3)).Return(v1.PodStatus{StartTime: &p3Time}, true)
resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
@ -208,7 +236,7 @@ func TestCadvisorListPodStats(t *testing.T) {
pods, err := p.ListPodStats()
assert.NoError(t, err)
assert.Equal(t, 3, len(pods))
assert.Equal(t, 4, len(pods))
indexPods := make(map[statsapi.PodReference]statsapi.PodStats, len(pods))
for _, pod := range pods {
indexPods[pod.PodRef] = pod
@ -261,6 +289,24 @@ func TestCadvisorListPodStats(t *testing.T) {
checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU)
checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory)
checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
// Validate Pod3 Results
ps, found = indexPods[prf3]
assert.True(t, found)
assert.Len(t, ps.Containers, 2)
indexCon = make(map[string]statsapi.ContainerStats, len(ps.Containers))
for _, con := range ps.Containers {
indexCon[con.Name] = con
}
con = indexCon[cName31]
assert.Equal(t, cName31, con.Name)
checkCPUStats(t, "Pod3Container1", seedPod3Container1, con.CPU)
checkMemoryStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Memory)
con = indexCon[cName30]
assert.Equal(t, cName30, con.Name)
checkEmptyCPUStats(t, "Pod3Container0", seedPod3Container0, con.CPU)
checkEmptyMemoryStats(t, "Pod3Container0", seedPod3Container0, infos["/pod3-c0-init"], con.Memory)
}
func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {

13
pkg/kubelet/stats/helper.go
View File

@ -40,10 +40,12 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
}
var cpuStats *statsapi.CPUStats
var memoryStats *statsapi.MemoryStats
cpuStats = &statsapi.CPUStats{
Time: metav1.NewTime(cstat.Timestamp),
UsageNanoCores: uint64Ptr(0),
UsageCoreNanoSeconds: uint64Ptr(0),
}
if info.Spec.HasCpu {
cpuStats = &statsapi.CPUStats{
Time: metav1.NewTime(cstat.Timestamp),
}
if cstat.CpuInst != nil {
cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total
}
@ -67,6 +69,11 @@ func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsa
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
memoryStats.AvailableBytes = &availableBytes
}
} else {
memoryStats = &statsapi.MemoryStats{
Time: metav1.NewTime(cstat.Timestamp),
WorkingSetBytes: uint64Ptr(0),
}
}
return cpuStats, memoryStats
}

25
pkg/kubelet/stats/stats_provider_test.go
View File

@ -621,7 +621,32 @@ func checkNetworkStats(t *testing.T, label string, seed int, stats *statsapi.Net
}
// container which had no stats should have zero-valued CPU usage
func checkEmptyCPUStats(t *testing.T, label string, seed int, stats *statsapi.CPUStats) {
require.NotNil(t, stats.Time, label+".CPU.Time")
require.NotNil(t, stats.UsageNanoCores, label+".CPU.UsageNanoCores")
require.NotNil(t, stats.UsageNanoCores, label+".CPU.UsageCoreSeconds")
assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".CPU.Time")
assert.EqualValues(t, 0, *stats.UsageNanoCores, label+".CPU.UsageCores")
assert.EqualValues(t, 0, *stats.UsageCoreNanoSeconds, label+".CPU.UsageCoreSeconds")
}
// container which had no stats should have zero-valued Memory usage
func checkEmptyMemoryStats(t *testing.T, label string, seed int, info cadvisorapiv2.ContainerInfo, stats *statsapi.MemoryStats) {
assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Mem.Time")
require.NotNil(t, stats.WorkingSetBytes, label+".Mem.WorkingSetBytes")
assert.EqualValues(t, 0, *stats.WorkingSetBytes, label+".Mem.WorkingSetBytes")
assert.Nil(t, stats.UsageBytes, label+".Mem.UsageBytes")
assert.Nil(t, stats.RSSBytes, label+".Mem.RSSBytes")
assert.Nil(t, stats.PageFaults, label+".Mem.PageFaults")
assert.Nil(t, stats.MajorPageFaults, label+".Mem.MajorPageFaults")
assert.Nil(t, stats.AvailableBytes, label+".Mem.AvailableBytes")
}
func checkCPUStats(t *testing.T, label string, seed int, stats *statsapi.CPUStats) {
require.NotNil(t, stats.Time, label+".CPU.Time")
require.NotNil(t, stats.UsageNanoCores, label+".CPU.UsageNanoCores")
require.NotNil(t, stats.UsageNanoCores, label+".CPU.UsageCoreSeconds")
assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".CPU.Time")
assert.EqualValues(t, seed+offsetCPUUsageCores, *stats.UsageNanoCores, label+".CPU.UsageCores")
assert.EqualValues(t, seed+offsetCPUUsageCoreSeconds, *stats.UsageCoreNanoSeconds, label+".CPU.UsageCoreSeconds")