github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 04:33:26 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #103424 from 249043822/br-cadvisor-perf

Browse Source 
Optimize kubelet stats provider for perfomace bottleneck
This commit is contained in:
Kubernetes Prow Robot 2021-09-21 11:56:18 -07:00 committed by GitHub
commit 44d4d007bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 40 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
pkg/kubelet/stats/cadvisor_stats_provider.go
View File

@ -91,12 +91,11 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
if err != nil {
return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
}
// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
allInfos := infos
infos = removeTerminatedContainerInfo(infos)
filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
// Map each container to a pod and update the PodStats with container data.
podToStats := map[statsapi.PodReference]*statsapi.PodStats{}
for key, cinfo := range infos {
for key, cinfo := range filteredInfos {
// On systemd using devicemapper each mount into the container has an
// associated cgroup. We ignore them to ensure we do not get duplicate
// entries in our summary. For details on .mount units:
@ -186,12 +185,10 @@ func (p *cadvisorStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats,
if err != nil {
return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
}
// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
allInfos := infos
infos = removeTerminatedContainerInfo(infos)
filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
// Map each container to a pod and update the PodStats with container data.
podToStats := map[statsapi.PodReference]*statsapi.PodStats{}
for key, cinfo := range infos {
for key, cinfo := range filteredInfos {
// On systemd using devicemapper each mount into the container has an
// associated cgroup. We ignore them to ensure we do not get duplicate
// entries in our summary. For details on .mount units:
@ -303,30 +300,32 @@ func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool {
// getCadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name
func getCadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo {
for key, info := range infos {
if cm.IsSystemdStyleName(key) {
// Convert to internal cgroup name and take the last component only.
internalCgroupName := cm.ParseSystemdToCgroupName(key)
key = internalCgroupName[len(internalCgroupName)-1]
} else {
// Take last component only.
key = path.Base(key)
}
if cm.GetPodCgroupNameSuffix(podUID) == key {
return &info
}
if info, found := infos[cm.GetPodCgroupNameSuffix(podUID)]; found {
return &info
}
return nil
}
// removeTerminatedContainerInfo returns the specified containerInfo but with
// the stats of the terminated containers removed.
//
// filterTerminatedContainerInfoAndAssembleByPodCgroupKey returns the specified containerInfo but with
// the stats of the terminated containers removed and all containerInfos assembled by pod cgroup key.
// the first return map is container cgroup name <-> ContainerInfo and
// the second return map is pod cgroup key <-> ContainerInfo.
// A ContainerInfo is considered to be of a terminated container if it has an
// older CreationTime and zero CPU instantaneous and memory RSS usage.
func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo {
func filterTerminatedContainerInfoAndAssembleByPodCgroupKey(containerInfo map[string]cadvisorapiv2.ContainerInfo) (map[string]cadvisorapiv2.ContainerInfo, map[string]cadvisorapiv2.ContainerInfo) {
cinfoMap := make(map[containerID][]containerInfoWithCgroup)
cinfosByPodCgroupKey := make(map[string]cadvisorapiv2.ContainerInfo)
for key, cinfo := range containerInfo {
var podCgroupKey string
if cm.IsSystemdStyleName(key) {
// Convert to internal cgroup name and take the last component only.
internalCgroupName := cm.ParseSystemdToCgroupName(key)
podCgroupKey = internalCgroupName[len(internalCgroupName)-1]
} else {
// Take last component only.
podCgroupKey = path.Base(key)
}
cinfosByPodCgroupKey[podCgroupKey] = cinfo
if !isPodManagedContainer(&cinfo) {
continue
}
@ -353,7 +352,7 @@ func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.Contai
}
}
}
return result
return result, cinfosByPodCgroupKey
}
// ByCreationTime implements sort.Interface for []containerInfoWithCgroup based

14
pkg/kubelet/stats/cadvisor_stats_provider_test.go
View File

@ -34,7 +34,7 @@ import (
statustest "k8s.io/kubernetes/pkg/kubelet/status/testing"
)
func TestRemoveTerminatedContainerInfo(t *testing.T) {
func TestFilterTerminatedContainerInfoAndAssembleByPodCgroupKey(t *testing.T) {
const (
seedPastPod0Infra = 1000
seedPastPod0Container0 = 2000
@ -63,10 +63,16 @@ func TestRemoveTerminatedContainerInfo(t *testing.T) {
"/pod0-i": getTestContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName),
"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace, cName00),
}
output := removeTerminatedContainerInfo(infos)
assert.Len(t, output, 2)
filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
assert.Len(t, filteredInfos, 2)
assert.Len(t, allInfos, 6)
for _, c := range []string{"/pod0-i", "/pod0-c0"} {
if _, found := output[c]; !found {
if _, found := filteredInfos[c]; !found {
t.Errorf("%q is expected to be in the output\n", c)
}
}
for _, c := range []string{"pod0-i-terminated-1", "pod0-c0-terminated-1", "pod0-i-terminated-2", "pod0-c0-terminated-2", "pod0-i", "pod0-c0"} {
if _, found := allInfos[c]; !found {
t.Errorf("%q is expected to be in the output\n", c)
}
}

12
pkg/kubelet/stats/cri_stats_provider.go
View File

@ -161,7 +161,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi
if err != nil {
return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
}
caInfos := getCRICadvisorStats(allInfos)
caInfos, allInfos := getCRICadvisorStats(allInfos)
// get network stats for containers.
// This is only used on Windows. For other platforms, (nil, nil) should be returned.
@ -255,7 +255,7 @@ func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, erro
if err != nil {
return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
}
caInfos := getCRICadvisorStats(allInfos)
caInfos, allInfos := getCRICadvisorStats(allInfos)
for _, stats := range resp {
containerID := stats.Attributes.Id
@ -811,10 +811,10 @@ func (p *criStatsProvider) addCadvisorContainerCPUAndMemoryStats(
}
}
func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo {
func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) (map[string]cadvisorapiv2.ContainerInfo, map[string]cadvisorapiv2.ContainerInfo) {
stats := make(map[string]cadvisorapiv2.ContainerInfo)
infos = removeTerminatedContainerInfo(infos)
for key, info := range infos {
filteredInfos, cinfosByPodCgroupKey := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
for key, info := range filteredInfos {
// On systemd using devicemapper each mount into the container has an
// associated cgroup. We ignore them to ensure we do not get duplicate
// entries in our summary. For details on .mount units:
@ -828,7 +828,7 @@ func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) map[strin
}
stats[extractIDFromCgroupPath(key)] = info
}
return stats
return stats, cinfosByPodCgroupKey
}
func extractIDFromCgroupPath(cgroupPath string) string {