mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Merge pull request #54606 from miaoyq/filter-out-duplicated-container-stats
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Filter out duplicated container stats **What this PR does / why we need it**: **Which issue this PR fixes** * fixes #53514 **Special notes for your reviewer**: /cc @Random-Liu Signed-off-by: Yanqiang Miao <miao.yanqiang@zte.com.cn>
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
5e508b37d9
@ -106,7 +106,7 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
||||
if !isPodManagedContainer(&cinfo) {
|
||||
continue
|
||||
}
|
||||
ref := buildPodRef(&cinfo)
|
||||
ref := buildPodRef(cinfo.Spec.Labels)
|
||||
|
||||
// Lookup the PodStats for the pod using the PodRef. If none exists,
|
||||
// initialize a new entry.
|
||||
@ -172,10 +172,10 @@ func (p *cadvisorStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
|
||||
}
|
||||
|
||||
// buildPodRef returns a PodReference that identifies the Pod managing cinfo
|
||||
func buildPodRef(cinfo *cadvisorapiv2.ContainerInfo) statsapi.PodReference {
|
||||
podName := kubetypes.GetPodName(cinfo.Spec.Labels)
|
||||
podNamespace := kubetypes.GetPodNamespace(cinfo.Spec.Labels)
|
||||
podUID := kubetypes.GetPodUID(cinfo.Spec.Labels)
|
||||
func buildPodRef(containerLabels map[string]string) statsapi.PodReference {
|
||||
podName := kubetypes.GetPodName(containerLabels)
|
||||
podNamespace := kubetypes.GetPodNamespace(containerLabels)
|
||||
podUID := kubetypes.GetPodUID(containerLabels)
|
||||
return statsapi.PodReference{Name: podName, Namespace: podNamespace, UID: podUID}
|
||||
}
|
||||
|
||||
@ -204,7 +204,7 @@ func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.Contai
|
||||
continue
|
||||
}
|
||||
cinfoID := containerID{
|
||||
podRef: buildPodRef(&cinfo),
|
||||
podRef: buildPodRef(cinfo.Spec.Labels),
|
||||
containerName: kubetypes.GetContainerName(cinfo.Spec.Labels),
|
||||
}
|
||||
cinfoMap[cinfoID] = append(cinfoMap[cinfoID], containerInfoWithCgroup{
|
||||
|
||||
@ -18,6 +18,7 @@ package stats
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
@ -32,6 +33,7 @@ import (
|
||||
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||
"k8s.io/kubernetes/pkg/kubelet/server/stats"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
)
|
||||
|
||||
// criStatsProvider implements the containerStatsProvider interface by getting
|
||||
@ -75,15 +77,10 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
||||
return nil, fmt.Errorf("failed to get rootFs info: %v", err)
|
||||
}
|
||||
|
||||
// Creates container map.
|
||||
containerMap := make(map[string]*runtimeapi.Container)
|
||||
containers, err := p.runtimeService.ListContainers(&runtimeapi.ContainerFilter{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list all containers: %v", err)
|
||||
}
|
||||
for _, c := range containers {
|
||||
containerMap[c.Id] = c
|
||||
}
|
||||
|
||||
// Creates pod sandbox map.
|
||||
podSandboxMap := make(map[string]*runtimeapi.PodSandbox)
|
||||
@ -107,6 +104,14 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list all container stats: %v", err)
|
||||
}
|
||||
|
||||
containers = removeTerminatedContainer(containers)
|
||||
// Creates container map.
|
||||
containerMap := make(map[string]*runtimeapi.Container)
|
||||
for _, c := range containers {
|
||||
containerMap[c.Id] = c
|
||||
}
|
||||
|
||||
for _, stats := range resp {
|
||||
containerID := stats.Attributes.Id
|
||||
container, found := containerMap[containerID]
|
||||
@ -286,3 +291,39 @@ func (p *criStatsProvider) makeContainerStats(
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// removeTerminatedContainer returns the specified container but with
|
||||
// the stats of the terminated containers removed.
|
||||
func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi.Container {
|
||||
containerMap := make(map[containerID][]*runtimeapi.Container)
|
||||
// Sort order by create time
|
||||
sort.Slice(containers, func(i, j int) bool {
|
||||
return containers[i].CreatedAt < containers[j].CreatedAt
|
||||
})
|
||||
for _, container := range containers {
|
||||
refID := containerID{
|
||||
podRef: buildPodRef(container.Labels),
|
||||
containerName: kubetypes.GetContainerName(container.Labels),
|
||||
}
|
||||
containerMap[refID] = append(containerMap[refID], container)
|
||||
}
|
||||
|
||||
result := make([]*runtimeapi.Container, 0)
|
||||
for _, refs := range containerMap {
|
||||
if len(refs) == 1 {
|
||||
result = append(result, refs[0])
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for i := 0; i < len(refs); i++ {
|
||||
if refs[i].State == runtimeapi.ContainerState_CONTAINER_RUNNING {
|
||||
found = true
|
||||
result = append(result, refs[i])
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
result = append(result, refs[len(refs)-1])
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@ -41,14 +41,20 @@ func TestCRIListPodStats(t *testing.T) {
|
||||
rootFsInfo = getTestFsInfo(1000)
|
||||
|
||||
sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
|
||||
container0 = makeFakeContainer(sandbox0, "container0-name")
|
||||
container0 = makeFakeContainer(sandbox0, "container0-name", 0, false)
|
||||
containerStats0 = makeFakeContainerStats(container0, imageFsStorageUUID)
|
||||
container1 = makeFakeContainer(sandbox0, "container1-name")
|
||||
container1 = makeFakeContainer(sandbox0, "container1-name", 0, false)
|
||||
containerStats1 = makeFakeContainerStats(container1, unknownStorageUUID)
|
||||
|
||||
sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
|
||||
container2 = makeFakeContainer(sandbox1, "container2-name")
|
||||
container2 = makeFakeContainer(sandbox1, "container2-name", 0, false)
|
||||
containerStats2 = makeFakeContainerStats(container2, imageFsStorageUUID)
|
||||
|
||||
sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
|
||||
container3 = makeFakeContainer(sandbox2, "container3-name", 0, true)
|
||||
containerStats3 = makeFakeContainerStats(container3, imageFsStorageUUID)
|
||||
container4 = makeFakeContainer(sandbox2, "container3-name", 1, false)
|
||||
containerStats4 = makeFakeContainerStats(container4, imageFsStorageUUID)
|
||||
)
|
||||
|
||||
var (
|
||||
@ -65,13 +71,13 @@ func TestCRIListPodStats(t *testing.T) {
|
||||
On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil).
|
||||
On("GetFsInfoByFsUUID", unknownStorageUUID).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice)
|
||||
fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{
|
||||
sandbox0, sandbox1,
|
||||
sandbox0, sandbox1, sandbox2,
|
||||
})
|
||||
fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
|
||||
container0, container1, container2,
|
||||
container0, container1, container2, container3, container4,
|
||||
})
|
||||
fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
|
||||
containerStats0, containerStats1, containerStats2,
|
||||
containerStats0, containerStats1, containerStats2, containerStats3, containerStats4,
|
||||
})
|
||||
|
||||
provider := NewCRIStatsProvider(
|
||||
@ -85,7 +91,7 @@ func TestCRIListPodStats(t *testing.T) {
|
||||
stats, err := provider.ListPodStats()
|
||||
assert := assert.New(t)
|
||||
assert.NoError(err)
|
||||
assert.Equal(2, len(stats))
|
||||
assert.Equal(3, len(stats))
|
||||
|
||||
podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats)
|
||||
for _, s := range stats {
|
||||
@ -100,26 +106,37 @@ func TestCRIListPodStats(t *testing.T) {
|
||||
for _, s := range p0.Containers {
|
||||
containerStatsMap[s.Name] = s
|
||||
}
|
||||
c1 := containerStatsMap["container0-name"]
|
||||
assert.Equal(container0.CreatedAt, c1.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c1, containerStats0)
|
||||
checkCRIRootfsStats(assert, c1, containerStats0, &imageFsInfo)
|
||||
c0 := containerStatsMap["container0-name"]
|
||||
assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c0, containerStats0)
|
||||
checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
|
||||
checkCRILogsStats(assert, c0, &rootFsInfo)
|
||||
c1 := containerStatsMap["container1-name"]
|
||||
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c1, containerStats1)
|
||||
checkCRIRootfsStats(assert, c1, containerStats1, nil)
|
||||
checkCRILogsStats(assert, c1, &rootFsInfo)
|
||||
c2 := containerStatsMap["container1-name"]
|
||||
assert.Equal(container1.CreatedAt, c2.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c2, containerStats1)
|
||||
checkCRIRootfsStats(assert, c2, containerStats1, nil)
|
||||
checkCRILogsStats(assert, c2, &rootFsInfo)
|
||||
|
||||
p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
|
||||
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
|
||||
assert.Equal(1, len(p1.Containers))
|
||||
|
||||
c3 := p1.Containers[0]
|
||||
assert.Equal("container2-name", c3.Name)
|
||||
assert.Equal(container2.CreatedAt, c3.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c3, containerStats2)
|
||||
checkCRIRootfsStats(assert, c3, containerStats2, &imageFsInfo)
|
||||
c2 := p1.Containers[0]
|
||||
assert.Equal("container2-name", c2.Name)
|
||||
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c2, containerStats2)
|
||||
checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
|
||||
checkCRILogsStats(assert, c2, &rootFsInfo)
|
||||
|
||||
p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
|
||||
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
|
||||
assert.Equal(1, len(p2.Containers))
|
||||
|
||||
c3 := p2.Containers[0]
|
||||
assert.Equal("container3-name", c3.Name)
|
||||
assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c3, containerStats4)
|
||||
checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
|
||||
checkCRILogsStats(assert, c3, &rootFsInfo)
|
||||
|
||||
mockCadvisor.AssertExpectations(t)
|
||||
@ -184,36 +201,38 @@ func makeFakePodSandbox(name, uid, namespace string) *critest.FakePodSandbox {
|
||||
return p
|
||||
}
|
||||
|
||||
func makeFakeContainer(sandbox *critest.FakePodSandbox, name string) *critest.FakeContainer {
|
||||
func makeFakeContainer(sandbox *critest.FakePodSandbox, name string, attempt uint32, terminated bool) *critest.FakeContainer {
|
||||
sandboxID := sandbox.PodSandboxStatus.Id
|
||||
c := &critest.FakeContainer{
|
||||
SandboxID: sandboxID,
|
||||
ContainerStatus: runtimeapi.ContainerStatus{
|
||||
Metadata: &runtimeapi.ContainerMetadata{Name: name},
|
||||
Metadata: &runtimeapi.ContainerMetadata{Name: name, Attempt: attempt},
|
||||
Image: &runtimeapi.ImageSpec{},
|
||||
ImageRef: "fake-image-ref",
|
||||
CreatedAt: time.Now().UnixNano(),
|
||||
State: runtimeapi.ContainerState_CONTAINER_RUNNING,
|
||||
},
|
||||
}
|
||||
c.ContainerStatus.Labels = map[string]string{
|
||||
"io.kubernetes.pod.name": sandbox.Metadata.Name,
|
||||
"io.kubernetes.pod.uid": sandbox.Metadata.Uid,
|
||||
"io.kubernetes.pod.namespace": sandbox.Metadata.Namespace,
|
||||
"io.kubernetes.container.name": name,
|
||||
}
|
||||
if terminated {
|
||||
c.ContainerStatus.State = runtimeapi.ContainerState_CONTAINER_EXITED
|
||||
} else {
|
||||
c.ContainerStatus.State = runtimeapi.ContainerState_CONTAINER_RUNNING
|
||||
}
|
||||
c.ContainerStatus.Id = critest.BuildContainerName(c.ContainerStatus.Metadata, sandboxID)
|
||||
return c
|
||||
}
|
||||
|
||||
func makeFakeContainerStats(container *critest.FakeContainer, imageFsUUID string) *runtimeapi.ContainerStats {
|
||||
return &runtimeapi.ContainerStats{
|
||||
containerStats := &runtimeapi.ContainerStats{
|
||||
Attributes: &runtimeapi.ContainerAttributes{
|
||||
Id: container.ContainerStatus.Id,
|
||||
Metadata: container.ContainerStatus.Metadata,
|
||||
},
|
||||
Cpu: &runtimeapi.CpuUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
},
|
||||
Memory: &runtimeapi.MemoryUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
WorkingSetBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
},
|
||||
WritableLayer: &runtimeapi.FilesystemUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
StorageId: &runtimeapi.StorageIdentifier{Uuid: imageFsUUID},
|
||||
@ -221,6 +240,20 @@ func makeFakeContainerStats(container *critest.FakeContainer, imageFsUUID string
|
||||
InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
},
|
||||
}
|
||||
if container.State == runtimeapi.ContainerState_CONTAINER_EXITED {
|
||||
containerStats.Cpu = nil
|
||||
containerStats.Memory = nil
|
||||
} else {
|
||||
containerStats.Cpu = &runtimeapi.CpuUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
}
|
||||
containerStats.Memory = &runtimeapi.MemoryUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
WorkingSetBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
}
|
||||
}
|
||||
return containerStats
|
||||
}
|
||||
|
||||
func makeFakeImageFsUsage(fsUUID string) *runtimeapi.FilesystemUsage {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user