mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 04:33:26 +00:00
Merge pull request #103424 from 249043822/br-cadvisor-perf
Optimize kubelet stats provider for perfomace bottleneck
This commit is contained in:
commit
44d4d007bf
@ -91,12 +91,11 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
|
||||
}
|
||||
// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
|
||||
allInfos := infos
|
||||
infos = removeTerminatedContainerInfo(infos)
|
||||
|
||||
filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
|
||||
// Map each container to a pod and update the PodStats with container data.
|
||||
podToStats := map[statsapi.PodReference]*statsapi.PodStats{}
|
||||
for key, cinfo := range infos {
|
||||
for key, cinfo := range filteredInfos {
|
||||
// On systemd using devicemapper each mount into the container has an
|
||||
// associated cgroup. We ignore them to ensure we do not get duplicate
|
||||
// entries in our summary. For details on .mount units:
|
||||
@ -186,12 +185,10 @@ func (p *cadvisorStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats,
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
|
||||
}
|
||||
// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
|
||||
allInfos := infos
|
||||
infos = removeTerminatedContainerInfo(infos)
|
||||
filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
|
||||
// Map each container to a pod and update the PodStats with container data.
|
||||
podToStats := map[statsapi.PodReference]*statsapi.PodStats{}
|
||||
for key, cinfo := range infos {
|
||||
for key, cinfo := range filteredInfos {
|
||||
// On systemd using devicemapper each mount into the container has an
|
||||
// associated cgroup. We ignore them to ensure we do not get duplicate
|
||||
// entries in our summary. For details on .mount units:
|
||||
@ -303,30 +300,32 @@ func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool {
|
||||
|
||||
// getCadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name
|
||||
func getCadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo {
|
||||
for key, info := range infos {
|
||||
if cm.IsSystemdStyleName(key) {
|
||||
// Convert to internal cgroup name and take the last component only.
|
||||
internalCgroupName := cm.ParseSystemdToCgroupName(key)
|
||||
key = internalCgroupName[len(internalCgroupName)-1]
|
||||
} else {
|
||||
// Take last component only.
|
||||
key = path.Base(key)
|
||||
}
|
||||
if cm.GetPodCgroupNameSuffix(podUID) == key {
|
||||
return &info
|
||||
}
|
||||
if info, found := infos[cm.GetPodCgroupNameSuffix(podUID)]; found {
|
||||
return &info
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeTerminatedContainerInfo returns the specified containerInfo but with
|
||||
// the stats of the terminated containers removed.
|
||||
//
|
||||
// filterTerminatedContainerInfoAndAssembleByPodCgroupKey returns the specified containerInfo but with
|
||||
// the stats of the terminated containers removed and all containerInfos assembled by pod cgroup key.
|
||||
// the first return map is container cgroup name <-> ContainerInfo and
|
||||
// the second return map is pod cgroup key <-> ContainerInfo.
|
||||
// A ContainerInfo is considered to be of a terminated container if it has an
|
||||
// older CreationTime and zero CPU instantaneous and memory RSS usage.
|
||||
func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo {
|
||||
func filterTerminatedContainerInfoAndAssembleByPodCgroupKey(containerInfo map[string]cadvisorapiv2.ContainerInfo) (map[string]cadvisorapiv2.ContainerInfo, map[string]cadvisorapiv2.ContainerInfo) {
|
||||
cinfoMap := make(map[containerID][]containerInfoWithCgroup)
|
||||
cinfosByPodCgroupKey := make(map[string]cadvisorapiv2.ContainerInfo)
|
||||
for key, cinfo := range containerInfo {
|
||||
var podCgroupKey string
|
||||
if cm.IsSystemdStyleName(key) {
|
||||
// Convert to internal cgroup name and take the last component only.
|
||||
internalCgroupName := cm.ParseSystemdToCgroupName(key)
|
||||
podCgroupKey = internalCgroupName[len(internalCgroupName)-1]
|
||||
} else {
|
||||
// Take last component only.
|
||||
podCgroupKey = path.Base(key)
|
||||
}
|
||||
cinfosByPodCgroupKey[podCgroupKey] = cinfo
|
||||
if !isPodManagedContainer(&cinfo) {
|
||||
continue
|
||||
}
|
||||
@ -353,7 +352,7 @@ func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.Contai
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
return result, cinfosByPodCgroupKey
|
||||
}
|
||||
|
||||
// ByCreationTime implements sort.Interface for []containerInfoWithCgroup based
|
||||
|
@ -34,7 +34,7 @@ import (
|
||||
statustest "k8s.io/kubernetes/pkg/kubelet/status/testing"
|
||||
)
|
||||
|
||||
func TestRemoveTerminatedContainerInfo(t *testing.T) {
|
||||
func TestFilterTerminatedContainerInfoAndAssembleByPodCgroupKey(t *testing.T) {
|
||||
const (
|
||||
seedPastPod0Infra = 1000
|
||||
seedPastPod0Container0 = 2000
|
||||
@ -63,10 +63,16 @@ func TestRemoveTerminatedContainerInfo(t *testing.T) {
|
||||
"/pod0-i": getTestContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName),
|
||||
"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace, cName00),
|
||||
}
|
||||
output := removeTerminatedContainerInfo(infos)
|
||||
assert.Len(t, output, 2)
|
||||
filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
|
||||
assert.Len(t, filteredInfos, 2)
|
||||
assert.Len(t, allInfos, 6)
|
||||
for _, c := range []string{"/pod0-i", "/pod0-c0"} {
|
||||
if _, found := output[c]; !found {
|
||||
if _, found := filteredInfos[c]; !found {
|
||||
t.Errorf("%q is expected to be in the output\n", c)
|
||||
}
|
||||
}
|
||||
for _, c := range []string{"pod0-i-terminated-1", "pod0-c0-terminated-1", "pod0-i-terminated-2", "pod0-c0-terminated-2", "pod0-i", "pod0-c0"} {
|
||||
if _, found := allInfos[c]; !found {
|
||||
t.Errorf("%q is expected to be in the output\n", c)
|
||||
}
|
||||
}
|
||||
|
@ -161,7 +161,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
|
||||
}
|
||||
caInfos := getCRICadvisorStats(allInfos)
|
||||
caInfos, allInfos := getCRICadvisorStats(allInfos)
|
||||
|
||||
// get network stats for containers.
|
||||
// This is only used on Windows. For other platforms, (nil, nil) should be returned.
|
||||
@ -255,7 +255,7 @@ func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, erro
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
|
||||
}
|
||||
caInfos := getCRICadvisorStats(allInfos)
|
||||
caInfos, allInfos := getCRICadvisorStats(allInfos)
|
||||
|
||||
for _, stats := range resp {
|
||||
containerID := stats.Attributes.Id
|
||||
@ -811,10 +811,10 @@ func (p *criStatsProvider) addCadvisorContainerCPUAndMemoryStats(
|
||||
}
|
||||
}
|
||||
|
||||
func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo {
|
||||
func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) (map[string]cadvisorapiv2.ContainerInfo, map[string]cadvisorapiv2.ContainerInfo) {
|
||||
stats := make(map[string]cadvisorapiv2.ContainerInfo)
|
||||
infos = removeTerminatedContainerInfo(infos)
|
||||
for key, info := range infos {
|
||||
filteredInfos, cinfosByPodCgroupKey := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
|
||||
for key, info := range filteredInfos {
|
||||
// On systemd using devicemapper each mount into the container has an
|
||||
// associated cgroup. We ignore them to ensure we do not get duplicate
|
||||
// entries in our summary. For details on .mount units:
|
||||
@ -828,7 +828,7 @@ func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) map[strin
|
||||
}
|
||||
stats[extractIDFromCgroupPath(key)] = info
|
||||
}
|
||||
return stats
|
||||
return stats, cinfosByPodCgroupKey
|
||||
}
|
||||
|
||||
func extractIDFromCgroupPath(cgroupPath string) string {
|
||||
|
Loading…
Reference in New Issue
Block a user