diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index f3cef84cd12..dc8ba6e46a7 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -256,13 +256,17 @@ func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Vo return map[string]volume.Volume{}, true } -func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error) { return nil, nil } -func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error) { return nil, nil } -func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error) { return nil, nil } -func (_ *fakeKubelet) RlimitStats() (*statsapi.RlimitStats, error) { return nil, nil } +func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error) { return nil, nil } +func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error) { return nil, nil } +func (_ *fakeKubelet) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) { return nil, nil } +func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error) { return nil, nil } +func (_ *fakeKubelet) RlimitStats() (*statsapi.RlimitStats, error) { return nil, nil } func (_ *fakeKubelet) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) { return nil, nil, nil } +func (_ *fakeKubelet) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error) { + return nil, nil +} type fakeAuth struct { authenticateFunc func(*http.Request) (user.Info, bool, error) diff --git a/pkg/kubelet/server/stats/handler.go b/pkg/kubelet/server/stats/handler.go index 6e665344503..683f20c32f3 100644 --- a/pkg/kubelet/server/stats/handler.go +++ b/pkg/kubelet/server/stats/handler.go @@ -43,6 +43,8 @@ type StatsProvider interface { // // ListPodStats returns the stats of all the containers managed by pods. ListPodStats() ([]statsapi.PodStats, error) + // ListPodCPUAndMemoryStats returns the CPU and memory stats of all the containers managed by pods. + ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) // ImageFsStats returns the stats of the image filesystem. ImageFsStats() (*statsapi.FsStats, error) @@ -51,6 +53,9 @@ type StatsProvider interface { // GetCgroupStats returns the stats and the networking usage of the cgroup // with the specified cgroupName. GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) + // GetCgroupCPUAndMemoryStats returns the CPU and memory stats of the cgroup with the specified cgroupName. + GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error) + // RootFsStats returns the stats of the node root filesystem. RootFsStats() (*statsapi.FsStats, error) diff --git a/pkg/kubelet/server/stats/summary.go b/pkg/kubelet/server/stats/summary.go index 2897aff50f5..fb646c5d2f3 100644 --- a/pkg/kubelet/server/stats/summary.go +++ b/pkg/kubelet/server/stats/summary.go @@ -91,30 +91,33 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error) } func (sp *summaryProviderImpl) GetCPUAndMemoryStats() (*statsapi.Summary, error) { - summary, err := sp.Get(false) + // TODO(timstclair): Consider returning a best-effort response if any of + // the following errors occur. + node, err := sp.provider.GetNode() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get node info: %v", err) } - summary.Node.Network = nil - summary.Node.Fs = nil - summary.Node.Runtime = nil - summary.Node.Rlimit = nil - for i := 0; i < len(summary.Node.SystemContainers); i++ { - summary.Node.SystemContainers[i].Accelerators = nil - summary.Node.SystemContainers[i].Rootfs = nil - summary.Node.SystemContainers[i].Logs = nil - summary.Node.SystemContainers[i].UserDefinedMetrics = nil + nodeConfig := sp.provider.GetNodeConfig() + rootStats, err := sp.provider.GetCgroupCPUAndMemoryStats("/", false) + if err != nil { + return nil, fmt.Errorf("failed to get root cgroup stats: %v", err) } - for i := 0; i < len(summary.Pods); i++ { - summary.Pods[i].Network = nil - summary.Pods[i].VolumeStats = nil - summary.Pods[i].EphemeralStorage = nil - for j := 0; j < len(summary.Pods[i].Containers); j++ { - summary.Pods[i].Containers[j].Accelerators = nil - summary.Pods[i].Containers[j].Rootfs = nil - summary.Pods[i].Containers[j].Logs = nil - summary.Pods[i].Containers[j].UserDefinedMetrics = nil - } + + podStats, err := sp.provider.ListPodCPUAndMemoryStats() + if err != nil { + return nil, fmt.Errorf("failed to list pod stats: %v", err) } - return summary, nil + + nodeStats := statsapi.NodeStats{ + NodeName: node.Name, + CPU: rootStats.CPU, + Memory: rootStats.Memory, + StartTime: rootStats.StartTime, + SystemContainers: sp.GetSystemContainersCPUAndMemoryStats(nodeConfig, podStats, false), + } + summary := statsapi.Summary{ + Node: nodeStats, + Pods: podStats, + } + return &summary, nil } diff --git a/pkg/kubelet/server/stats/summary_sys_containers.go b/pkg/kubelet/server/stats/summary_sys_containers.go index 7c4205c93b3..7179e828020 100644 --- a/pkg/kubelet/server/stats/summary_sys_containers.go +++ b/pkg/kubelet/server/stats/summary_sys_containers.go @@ -53,3 +53,30 @@ func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig return stats } + +func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) { + systemContainers := map[string]struct { + name string + forceStatsUpdate bool + }{ + statsapi.SystemContainerKubelet: {nodeConfig.KubeletCgroupsName, false}, + statsapi.SystemContainerRuntime: {nodeConfig.RuntimeCgroupsName, false}, + statsapi.SystemContainerMisc: {nodeConfig.SystemCgroupsName, false}, + statsapi.SystemContainerPods: {sp.provider.GetPodCgroupRoot(), updateStats}, + } + for sys, cont := range systemContainers { + // skip if cgroup name is undefined (not all system containers are required) + if cont.name == "" { + continue + } + s, err := sp.provider.GetCgroupCPUAndMemoryStats(cont.name, cont.forceStatsUpdate) + if err != nil { + glog.Errorf("Failed to get system container stats for %q: %v", cont.name, err) + continue + } + s.Name = sys + stats = append(stats, *s) + } + + return stats +} diff --git a/pkg/kubelet/server/stats/summary_sys_containers_windows.go b/pkg/kubelet/server/stats/summary_sys_containers_windows.go index b5e3affb7b3..cb8dcb3c898 100644 --- a/pkg/kubelet/server/stats/summary_sys_containers_windows.go +++ b/pkg/kubelet/server/stats/summary_sys_containers_windows.go @@ -27,11 +27,16 @@ import ( ) func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) { - stats = append(stats, sp.getSystemPodsStats(nodeConfig, podStats, updateStats)) + stats = append(stats, sp.getSystemPodsCPUAndMemoryStats(nodeConfig, podStats, updateStats)) return stats } -func (sp *summaryProviderImpl) getSystemPodsStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) statsapi.ContainerStats { +func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) { + stats = append(stats, sp.getSystemPodsCPUAndMemoryStats(nodeConfig, podStats, updateStats)) + return stats +} + +func (sp *summaryProviderImpl) getSystemPodsCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) statsapi.ContainerStats { now := metav1.NewTime(time.Now()) podsSummary := statsapi.ContainerStats{ StartTime: now, diff --git a/pkg/kubelet/server/stats/summary_test.go b/pkg/kubelet/server/stats/summary_test.go index b6d7dbbab74..d210298f59f 100644 --- a/pkg/kubelet/server/stats/summary_test.go +++ b/pkg/kubelet/server/stats/summary_test.go @@ -33,15 +33,6 @@ import ( ) var ( - podStats = []statsapi.PodStats{ - { - PodRef: statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"}, - StartTime: metav1.NewTime(time.Now()), - Containers: []statsapi.ContainerStats{*getContainerStats()}, - Network: getNetworkStats(), - VolumeStats: []statsapi.VolumeStats{*getVolumeStats()}, - }, - } imageFsStats = getFsStats() rootFsStats = getFsStats() node = &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "test-node"}} @@ -50,8 +41,23 @@ var ( SystemCgroupsName: "/misc", KubeletCgroupsName: "/kubelet", } - cgroupRoot = "/kubepods" - cgroupStatsMap = map[string]struct { + cgroupRoot = "/kubepods" + rlimitStats = getRlimitStats() +) + +func TestSummaryProviderGetStats(t *testing.T) { + assert := assert.New(t) + + podStats := []statsapi.PodStats{ + { + PodRef: statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"}, + StartTime: metav1.NewTime(time.Now()), + Containers: []statsapi.ContainerStats{*getContainerStats()}, + Network: getNetworkStats(), + VolumeStats: []statsapi.VolumeStats{*getVolumeStats()}, + }, + } + cgroupStatsMap := map[string]struct { cs *statsapi.ContainerStats ns *statsapi.NetworkStats }{ @@ -61,11 +67,6 @@ var ( "/kubelet": {cs: getContainerStats(), ns: getNetworkStats()}, "/pods": {cs: getContainerStats(), ns: getNetworkStats()}, } - rlimitStats = getRlimitStats() -) - -func TestSummaryProviderGetStats(t *testing.T) { - assert := assert.New(t) mockStatsProvider := new(statstest.StatsProvider) mockStatsProvider. @@ -133,20 +134,34 @@ func TestSummaryProviderGetStats(t *testing.T) { func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) { assert := assert.New(t) + podStats := []statsapi.PodStats{ + { + PodRef: statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"}, + StartTime: metav1.NewTime(time.Now()), + Containers: []statsapi.ContainerStats{*getContainerStats()}, + }, + } + cgroupStatsMap := map[string]struct { + cs *statsapi.ContainerStats + }{ + "/": {cs: getVolumeCPUAndMemoryStats()}, + "/runtime": {cs: getVolumeCPUAndMemoryStats()}, + "/misc": {cs: getVolumeCPUAndMemoryStats()}, + "/kubelet": {cs: getVolumeCPUAndMemoryStats()}, + "/pods": {cs: getVolumeCPUAndMemoryStats()}, + } + mockStatsProvider := new(statstest.StatsProvider) mockStatsProvider. On("GetNode").Return(node, nil). On("GetNodeConfig").Return(nodeConfig). On("GetPodCgroupRoot").Return(cgroupRoot). - On("ListPodStats").Return(podStats, nil). - On("ImageFsStats").Return(imageFsStats, nil). - On("RootFsStats").Return(rootFsStats, nil). - On("RlimitStats").Return(rlimitStats, nil). - On("GetCgroupStats", "/", false).Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil). - On("GetCgroupStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil). - On("GetCgroupStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil). - On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil). - On("GetCgroupStats", "/kubepods", false).Return(cgroupStatsMap["/pods"].cs, cgroupStatsMap["/pods"].ns, nil) + On("ListPodCPUAndMemoryStats").Return(podStats, nil). + On("GetCgroupCPUAndMemoryStats", "/", false).Return(cgroupStatsMap["/"].cs, nil). + On("GetCgroupCPUAndMemoryStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, nil). + On("GetCgroupCPUAndMemoryStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, nil). + On("GetCgroupCPUAndMemoryStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, nil). + On("GetCgroupCPUAndMemoryStats", "/kubepods", false).Return(cgroupStatsMap["/pods"].cs, nil) provider := NewSummaryProvider(mockStatsProvider) summary, err := provider.GetCPUAndMemoryStats() @@ -201,6 +216,15 @@ func getContainerStats() *statsapi.ContainerStats { f.Fuzz(v) return v } +func getVolumeCPUAndMemoryStats() *statsapi.ContainerStats { + f := fuzz.New().NilChance(0) + v := &statsapi.ContainerStats{} + f.Fuzz(&v.Name) + f.Fuzz(&v.StartTime) + f.Fuzz(v.CPU) + f.Fuzz(v.Memory) + return v +} func getVolumeStats() *statsapi.VolumeStats { f := fuzz.New().NilChance(0) diff --git a/pkg/kubelet/server/stats/testing/mock_stats_provider.go b/pkg/kubelet/server/stats/testing/mock_stats_provider.go index a50ad43b375..9db0482ab39 100644 --- a/pkg/kubelet/server/stats/testing/mock_stats_provider.go +++ b/pkg/kubelet/server/stats/testing/mock_stats_provider.go @@ -64,6 +64,29 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*v return r0, r1, r2 } +// GetCgroupCPUAndMemoryStats provides a mock function with given fields: cgroupName, updateStats +func (_m *StatsProvider) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*v1alpha1.ContainerStats, error) { + ret := _m.Called(cgroupName, updateStats) + + var r0 *v1alpha1.ContainerStats + if rf, ok := ret.Get(0).(func(string, bool) *v1alpha1.ContainerStats); ok { + r0 = rf(cgroupName, updateStats) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.ContainerStats) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, bool) error); ok { + r1 = rf(cgroupName, updateStats) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetPodByCgroupfs provides the pod that maps to the specified cgroup, as well // as whether the pod was found. func (_m *StatsProvider) GetPodByCgroupfs(cgroupfs string) (*corev1.Pod, bool) { @@ -252,6 +275,29 @@ func (_m *StatsProvider) ListPodStats() ([]v1alpha1.PodStats, error) { return r0, r1 } +// ListPodCPUAndMemoryStats provides a mock function with given fields: +func (_m *StatsProvider) ListPodCPUAndMemoryStats() ([]v1alpha1.PodStats, error) { + ret := _m.Called() + + var r0 []v1alpha1.PodStats + if rf, ok := ret.Get(0).(func() []v1alpha1.PodStats); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]v1alpha1.PodStats) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ListVolumesForPod provides a mock function with given fields: podUID func (_m *StatsProvider) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) { ret := _m.Called(podUID) diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 8529872cd50..35820f7e94a 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -145,6 +145,68 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { return result, nil } +// ListPodCPUAndMemoryStats returns the cpu and memory stats of all the pod-managed containers. +func (p *cadvisorStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) { + infos, err := getCadvisorContainerInfo(p.cadvisor) + if err != nil { + return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err) + } + // removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first + allInfos := infos + infos = removeTerminatedContainerInfo(infos) + // Map each container to a pod and update the PodStats with container data. + podToStats := map[statsapi.PodReference]*statsapi.PodStats{} + for key, cinfo := range infos { + // On systemd using devicemapper each mount into the container has an + // associated cgroup. We ignore them to ensure we do not get duplicate + // entries in our summary. For details on .mount units: + // http://man7.org/linux/man-pages/man5/systemd.mount.5.html + if strings.HasSuffix(key, ".mount") { + continue + } + // Build the Pod key if this container is managed by a Pod + if !isPodManagedContainer(&cinfo) { + continue + } + ref := buildPodRef(cinfo.Spec.Labels) + + // Lookup the PodStats for the pod using the PodRef. If none exists, + // initialize a new entry. + podStats, found := podToStats[ref] + if !found { + podStats = &statsapi.PodStats{PodRef: ref} + podToStats[ref] = podStats + } + + // Update the PodStats entry with the stats from the container by + // adding it to podStats.Containers. + containerName := kubetypes.GetContainerName(cinfo.Spec.Labels) + if containerName == leaky.PodInfraContainerName { + // Special case for infrastructure container which is hidden from + // the user and has network stats. + podStats.StartTime = metav1.NewTime(cinfo.Spec.CreationTime) + } else { + podStats.Containers = append(podStats.Containers, *cadvisorInfoToContainerCPUAndMemoryStats(containerName, &cinfo)) + } + } + + // Add each PodStats to the result. + result := make([]statsapi.PodStats, 0, len(podToStats)) + for _, podStats := range podToStats { + podUID := types.UID(podStats.PodRef.UID) + // Lookup the pod-level cgroup's CPU and memory stats + podInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos) + if podInfo != nil { + cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo) + podStats.CPU = cpu + podStats.Memory = memory + } + result = append(result, *podStats) + } + + return result, nil +} + func calcEphemeralStorage(containers []statsapi.ContainerStats, volumes []statsapi.VolumeStats, rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.FsStats { result := &statsapi.FsStats{ Time: metav1.NewTime(rootFsInfo.Timestamp), diff --git a/pkg/kubelet/stats/cadvisor_stats_provider_test.go b/pkg/kubelet/stats/cadvisor_stats_provider_test.go index 942fa2e2c50..68a41acb1b9 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider_test.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider_test.go @@ -258,6 +258,169 @@ func TestCadvisorListPodStats(t *testing.T) { checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network) } +func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) { + const ( + namespace0 = "test0" + namespace2 = "test2" + ) + const ( + seedRoot = 0 + seedRuntime = 100 + seedKubelet = 200 + seedMisc = 300 + seedPod0Infra = 1000 + seedPod0Container0 = 2000 + seedPod0Container1 = 2001 + seedPod1Infra = 3000 + seedPod1Container = 4000 + seedPod2Infra = 5000 + seedPod2Container = 6000 + seedEphemeralVolume1 = 10000 + seedEphemeralVolume2 = 10001 + seedPersistentVolume1 = 20000 + seedPersistentVolume2 = 20001 + ) + const ( + pName0 = "pod0" + pName1 = "pod1" + pName2 = "pod0" // ensure pName2 conflicts with pName0, but is in a different namespace + ) + const ( + cName00 = "c0" + cName01 = "c1" + cName10 = "c0" // ensure cName10 conflicts with cName02, but is in a different pod + cName20 = "c1" // ensure cName20 conflicts with cName01, but is in a different pod + namespace + ) + + prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0} + prf1 := statsapi.PodReference{Name: pName1, Namespace: namespace0, UID: "UID" + pName1} + prf2 := statsapi.PodReference{Name: pName2, Namespace: namespace2, UID: "UID" + pName2} + infos := map[string]cadvisorapiv2.ContainerInfo{ + "/": getTestContainerInfo(seedRoot, "", "", ""), + "/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""), + "/kubelet": getTestContainerInfo(seedKubelet, "", "", ""), + "/system": getTestContainerInfo(seedMisc, "", "", ""), + // Pod0 - Namespace0 + "/pod0-i": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName), + "/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00), + "/pod0-c1": getTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01), + // Pod1 - Namespace0 + "/pod1-i": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName), + "/pod1-c0": getTestContainerInfo(seedPod1Container, pName1, namespace0, cName10), + // Pod2 - Namespace2 + "/pod2-i": getTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName), + "/pod2-c0": getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20), + "/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName), + "/kubepods/podUIDpod1": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName), + } + + // memory limit overrides for each container (used to test available bytes if a memory limit is known) + memoryLimitOverrides := map[string]uint64{ + "/": uint64(1 << 30), + "/pod2-c0": uint64(1 << 15), + } + for name, memoryLimitOverride := range memoryLimitOverrides { + info, found := infos[name] + if !found { + t.Errorf("No container defined with name %v", name) + } + info.Spec.Memory.Limit = memoryLimitOverride + infos[name] = info + } + + options := cadvisorapiv2.RequestOptions{ + IdType: cadvisorapiv2.TypeName, + Count: 2, + Recursive: true, + } + + mockCadvisor := new(cadvisortest.Mock) + mockCadvisor. + On("ContainerInfoV2", "/", options).Return(infos, nil) + + ephemeralVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedEphemeralVolume1, "ephemeralVolume1"), + getPodVolumeStats(seedEphemeralVolume2, "ephemeralVolume2")} + persistentVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedPersistentVolume1, "persistentVolume1"), + getPodVolumeStats(seedPersistentVolume2, "persistentVolume2")} + volumeStats := serverstats.PodVolumeStats{ + EphemeralVolumes: ephemeralVolumes, + PersistentVolumes: persistentVolumes, + } + + resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats} + + p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, nil) + pods, err := p.ListPodCPUAndMemoryStats() + assert.NoError(t, err) + + assert.Equal(t, 3, len(pods)) + indexPods := make(map[statsapi.PodReference]statsapi.PodStats, len(pods)) + for _, pod := range pods { + indexPods[pod.PodRef] = pod + } + + // Validate Pod0 Results + ps, found := indexPods[prf0] + assert.True(t, found) + assert.Len(t, ps.Containers, 2) + indexCon := make(map[string]statsapi.ContainerStats, len(ps.Containers)) + for _, con := range ps.Containers { + indexCon[con.Name] = con + } + con := indexCon[cName00] + assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix()) + checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU) + checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory) + assert.Nil(t, con.Rootfs) + assert.Nil(t, con.Logs) + assert.Nil(t, con.Accelerators) + assert.Nil(t, con.UserDefinedMetrics) + + con = indexCon[cName01] + assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix()) + checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU) + checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory) + assert.Nil(t, con.Rootfs) + assert.Nil(t, con.Logs) + assert.Nil(t, con.Accelerators) + assert.Nil(t, con.UserDefinedMetrics) + + assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix()) + assert.Nil(t, ps.EphemeralStorage) + assert.Nil(t, ps.VolumeStats) + assert.Nil(t, ps.Network) + if ps.CPU != nil { + checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU) + } + if ps.Memory != nil { + checkMemoryStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Memory) + } + + // Validate Pod1 Results + ps, found = indexPods[prf1] + assert.True(t, found) + assert.Len(t, ps.Containers, 1) + con = ps.Containers[0] + assert.Equal(t, cName10, con.Name) + checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU) + checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory) + assert.Nil(t, ps.EphemeralStorage) + assert.Nil(t, ps.VolumeStats) + assert.Nil(t, ps.Network) + + // Validate Pod2 Results + ps, found = indexPods[prf2] + assert.True(t, found) + assert.Len(t, ps.Containers, 1) + con = ps.Containers[0] + assert.Equal(t, cName20, con.Name) + checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU) + checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory) + assert.Nil(t, ps.EphemeralStorage) + assert.Nil(t, ps.VolumeStats) + assert.Nil(t, ps.Network) +} + func TestCadvisorImagesFsStats(t *testing.T) { var ( assert = assert.New(t) diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index 6580f46ead0..3ebf3a0fec2 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -169,6 +169,87 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { return result, nil } +// ListPodCPUAndMemoryStats returns the CPU and Memory stats of all the pod-managed containers. +func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) { + containers, err := p.runtimeService.ListContainers(&runtimeapi.ContainerFilter{}) + if err != nil { + return nil, fmt.Errorf("failed to list all containers: %v", err) + } + + // Creates pod sandbox map. + podSandboxMap := make(map[string]*runtimeapi.PodSandbox) + podSandboxes, err := p.runtimeService.ListPodSandbox(&runtimeapi.PodSandboxFilter{}) + if err != nil { + return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err) + } + for _, s := range podSandboxes { + podSandboxMap[s.Id] = s + } + + // sandboxIDToPodStats is a temporary map from sandbox ID to its pod stats. + sandboxIDToPodStats := make(map[string]*statsapi.PodStats) + + resp, err := p.runtimeService.ListContainerStats(&runtimeapi.ContainerStatsFilter{}) + if err != nil { + return nil, fmt.Errorf("failed to list all container stats: %v", err) + } + + containers = removeTerminatedContainer(containers) + // Creates container map. + containerMap := make(map[string]*runtimeapi.Container) + for _, c := range containers { + containerMap[c.Id] = c + } + + allInfos, err := getCadvisorContainerInfo(p.cadvisor) + if err != nil { + return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err) + } + caInfos := getCRICadvisorStats(allInfos) + + for _, stats := range resp { + containerID := stats.Attributes.Id + container, found := containerMap[containerID] + if !found { + continue + } + + podSandboxID := container.PodSandboxId + podSandbox, found := podSandboxMap[podSandboxID] + if !found { + continue + } + + // Creates the stats of the pod (if not created yet) which the + // container belongs to. + ps, found := sandboxIDToPodStats[podSandboxID] + if !found { + ps = buildPodStats(podSandbox) + sandboxIDToPodStats[podSandboxID] = ps + } + + // Fill available CPU and memory stats for full set of required pod stats + cs := p.makeContainerCPUAndMemoryStats(stats, container) + p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs) + + // If cadvisor stats is available for the container, use it to populate + // container stats + caStats, caFound := caInfos[containerID] + if !caFound { + glog.V(4).Infof("Unable to find cadvisor stats for %q", containerID) + } else { + p.addCadvisorContainerStats(cs, &caStats) + } + ps.Containers = append(ps.Containers, *cs) + } + + result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats)) + for _, s := range sandboxIDToPodStats { + result = append(result, *s) + } + return result, nil +} + // ImageFsStats returns the stats of the image filesystem. func (p *criStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { resp, err := p.imageService.ImageFsInfo() @@ -393,6 +474,33 @@ func (p *criStatsProvider) makeContainerStats( return result } +func (p *criStatsProvider) makeContainerCPUAndMemoryStats( + stats *runtimeapi.ContainerStats, + container *runtimeapi.Container, +) *statsapi.ContainerStats { + result := &statsapi.ContainerStats{ + Name: stats.Attributes.Metadata.Name, + // The StartTime in the summary API is the container creation time. + StartTime: metav1.NewTime(time.Unix(0, container.CreatedAt)), + CPU: &statsapi.CPUStats{}, + Memory: &statsapi.MemoryStats{}, + // UserDefinedMetrics is not supported by CRI. + } + if stats.Cpu != nil { + result.CPU.Time = metav1.NewTime(time.Unix(0, stats.Cpu.Timestamp)) + if stats.Cpu.UsageCoreNanoSeconds != nil { + result.CPU.UsageCoreNanoSeconds = &stats.Cpu.UsageCoreNanoSeconds.Value + } + } + if stats.Memory != nil { + result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp)) + if stats.Memory.WorkingSetBytes != nil { + result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value + } + } + return result +} + // removeTerminatedContainer returns the specified container but with // the stats of the terminated containers removed. func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi.Container { diff --git a/pkg/kubelet/stats/cri_stats_provider_test.go b/pkg/kubelet/stats/cri_stats_provider_test.go index 9e1dd1bb8c3..3e73625a2f9 100644 --- a/pkg/kubelet/stats/cri_stats_provider_test.go +++ b/pkg/kubelet/stats/cri_stats_provider_test.go @@ -46,33 +46,33 @@ const ( offsetUsage ) +const ( + seedRoot = 0 + seedKubelet = 200 + seedMisc = 300 + seedSandbox0 = 1000 + seedContainer0 = 2000 + seedSandbox1 = 3000 + seedContainer1 = 4000 + seedContainer2 = 5000 + seedSandbox2 = 6000 + seedContainer3 = 7000 +) + +const ( + pName0 = "pod0" + pName1 = "pod1" + pName2 = "pod2" +) + +const ( + cName0 = "container0-name" + cName1 = "container1-name" + cName2 = "container2-name" + cName3 = "container3-name" +) + func TestCRIListPodStats(t *testing.T) { - const ( - seedRoot = 0 - seedKubelet = 200 - seedMisc = 300 - seedSandbox0 = 1000 - seedContainer0 = 2000 - seedSandbox1 = 3000 - seedContainer1 = 4000 - seedContainer2 = 5000 - seedSandbox2 = 6000 - seedContainer3 = 7000 - ) - - const ( - pName0 = "pod0" - pName1 = "pod1" - pName2 = "pod2" - ) - - const ( - cName0 = "container0-name" - cName1 = "container1-name" - cName2 = "container2-name" - cName3 = "container3-name" - ) - var ( imageFsMountpoint = "/test/mount/point" unknownMountpoint = "/unknown/mount/point" @@ -242,6 +242,166 @@ func TestCRIListPodStats(t *testing.T) { mockCadvisor.AssertExpectations(t) } +func TestCRIListPodCPUAndMemoryStats(t *testing.T) { + + var ( + imageFsMountpoint = "/test/mount/point" + unknownMountpoint = "/unknown/mount/point" + + sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns") + sandbox0Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox0.PodSandboxStatus.Metadata.Uid)) + container0 = makeFakeContainer(sandbox0, cName0, 0, false) + containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint) + container1 = makeFakeContainer(sandbox0, cName1, 0, false) + containerStats1 = makeFakeContainerStats(container1, unknownMountpoint) + + sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns") + sandbox1Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox1.PodSandboxStatus.Metadata.Uid)) + container2 = makeFakeContainer(sandbox1, cName2, 0, false) + containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint) + + sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns") + sandbox2Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox2.PodSandboxStatus.Metadata.Uid)) + container3 = makeFakeContainer(sandbox2, cName3, 0, true) + containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint) + container4 = makeFakeContainer(sandbox2, cName3, 1, false) + containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint) + ) + + var ( + mockCadvisor = new(cadvisortest.Mock) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + mockPodManager = new(kubepodtest.MockManager) + resourceAnalyzer = new(fakeResourceAnalyzer) + fakeRuntimeService = critest.NewFakeRuntimeService() + ) + + infos := map[string]cadvisorapiv2.ContainerInfo{ + "/": getTestContainerInfo(seedRoot, "", "", ""), + "/kubelet": getTestContainerInfo(seedKubelet, "", "", ""), + "/system": getTestContainerInfo(seedMisc, "", "", ""), + sandbox0.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName), + sandbox0Cgroup: getTestContainerInfo(seedSandbox0, "", "", ""), + container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0), + container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1), + sandbox1.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox1, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName), + sandbox1Cgroup: getTestContainerInfo(seedSandbox1, "", "", ""), + container2.ContainerStatus.Id: getTestContainerInfo(seedContainer2, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, cName2), + sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName), + sandbox2Cgroup: getTestContainerInfo(seedSandbox2, "", "", ""), + container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3), + } + + options := cadvisorapiv2.RequestOptions{ + IdType: cadvisorapiv2.TypeName, + Count: 2, + Recursive: true, + } + + mockCadvisor. + On("ContainerInfoV2", "/", options).Return(infos, nil) + fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{ + sandbox0, sandbox1, sandbox2, + }) + fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{ + container0, container1, container2, container3, container4, + }) + fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{ + containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, + }) + + ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"}) + persistentVolumes := makeFakeVolumeStats([]string{"persisVolume1, persisVolumes2"}) + resourceAnalyzer.podVolumeStats = serverstats.PodVolumeStats{ + EphemeralVolumes: ephemeralVolumes, + PersistentVolumes: persistentVolumes, + } + + provider := NewCRIStatsProvider( + mockCadvisor, + resourceAnalyzer, + mockPodManager, + mockRuntimeCache, + fakeRuntimeService, + nil, + nil, + ) + + stats, err := provider.ListPodCPUAndMemoryStats() + assert := assert.New(t) + assert.NoError(err) + assert.Equal(3, len(stats)) + + podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats) + for _, s := range stats { + podStatsMap[s.PodRef] = s + } + + p0 := podStatsMap[statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}] + assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano()) + assert.Equal(2, len(p0.Containers)) + assert.Nil(p0.EphemeralStorage) + assert.Nil(p0.VolumeStats) + assert.Nil(p0.Network) + checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0]) + + containerStatsMap := make(map[string]statsapi.ContainerStats) + for _, s := range p0.Containers { + containerStatsMap[s.Name] = s + } + + c0 := containerStatsMap[cName0] + assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano()) + checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0]) + assert.Nil(c0.Rootfs) + assert.Nil(c0.Logs) + assert.Nil(c0.Accelerators) + assert.Nil(c0.UserDefinedMetrics) + c1 := containerStatsMap[cName1] + assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano()) + checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0]) + assert.Nil(c1.Rootfs) + assert.Nil(c1.Logs) + assert.Nil(c1.Accelerators) + assert.Nil(c1.UserDefinedMetrics) + + p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}] + assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano()) + assert.Equal(1, len(p1.Containers)) + assert.Nil(p1.EphemeralStorage) + assert.Nil(p1.VolumeStats) + assert.Nil(p1.Network) + checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0]) + + c2 := p1.Containers[0] + assert.Equal(cName2, c2.Name) + assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano()) + checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0]) + assert.Nil(c2.Rootfs) + assert.Nil(c2.Logs) + assert.Nil(c2.Accelerators) + assert.Nil(c2.UserDefinedMetrics) + + p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}] + assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano()) + assert.Equal(1, len(p2.Containers)) + assert.Nil(p2.EphemeralStorage) + assert.Nil(p2.VolumeStats) + assert.Nil(p2.Network) + checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0]) + + c3 := p2.Containers[0] + assert.Equal(cName3, c3.Name) + assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano()) + checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0]) + assert.Nil(c2.Rootfs) + assert.Nil(c2.Logs) + assert.Nil(c2.Accelerators) + assert.Nil(c2.UserDefinedMetrics) + + mockCadvisor.AssertExpectations(t) +} + func TestCRIImagesFsStats(t *testing.T) { var ( imageFsMountpoint = "/test/mount/point" diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index e8917958ff6..2bdda4314a6 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -132,6 +132,21 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo return result } +// cadvisorInfoToContainerCPUAndMemoryStats returns the statsapi.ContainerStats converted +// from the container and filesystem info. +func cadvisorInfoToContainerCPUAndMemoryStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.ContainerStats { + result := &statsapi.ContainerStats{ + StartTime: metav1.NewTime(info.Spec.CreationTime), + Name: name, + } + + cpu, memory := cadvisorInfoToCPUandMemoryStats(info) + result.CPU = cpu + result.Memory = memory + + return result +} + // cadvisorInfoToNetworkStats returns the statsapi.NetworkStats converted from // the container info from cadvisor. func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.NetworkStats { diff --git a/pkg/kubelet/stats/stats_provider.go b/pkg/kubelet/stats/stats_provider.go index 29a24a1b3c2..903f8678a64 100644 --- a/pkg/kubelet/stats/stats_provider.go +++ b/pkg/kubelet/stats/stats_provider.go @@ -85,6 +85,7 @@ type StatsProvider struct { // containers managed by pods. type containerStatsProvider interface { ListPodStats() ([]statsapi.PodStats, error) + ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) ImageFsStats() (*statsapi.FsStats, error) ImageFsDevice() (string, error) } @@ -106,6 +107,18 @@ func (p *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*st return s, n, nil } +// GetCgroupCPUAndMemoryStats returns the CPU and memory stats of the cgroup with the cgroupName. Note that +// this function doesn't generate filesystem stats. +func (p *StatsProvider) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error) { + info, err := getCgroupInfo(p.cadvisor, cgroupName, updateStats) + if err != nil { + return nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err) + } + // Rootfs and imagefs doesn't make sense for raw cgroup. + s := cadvisorInfoToContainerCPUAndMemoryStats(cgroupName, info) + return s, nil +} + // RootFsStats returns the stats of the node root filesystem. func (p *StatsProvider) RootFsStats() (*statsapi.FsStats, error) { rootFsInfo, err := p.cadvisor.RootFsInfo() diff --git a/pkg/kubelet/stats/stats_provider_test.go b/pkg/kubelet/stats/stats_provider_test.go index bbb149075bf..26884547e51 100644 --- a/pkg/kubelet/stats/stats_provider_test.go +++ b/pkg/kubelet/stats/stats_provider_test.go @@ -100,6 +100,39 @@ func TestGetCgroupStats(t *testing.T) { mockCadvisor.AssertExpectations(t) } +func TestGetCgroupCPUAndMemoryStats(t *testing.T) { + const ( + cgroupName = "test-cgroup-name" + containerInfoSeed = 1000 + updateStats = false + ) + var ( + mockCadvisor = new(cadvisortest.Mock) + mockPodManager = new(kubepodtest.MockManager) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + + assert = assert.New(t) + options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false} + + containerInfo = getTestContainerInfo(containerInfoSeed, "test-pod", "test-ns", "test-container") + containerInfoMap = map[string]cadvisorapiv2.ContainerInfo{cgroupName: containerInfo} + ) + + mockCadvisor.On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil) + + provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{}) + cs, err := provider.GetCgroupCPUAndMemoryStats(cgroupName, updateStats) + assert.NoError(err) + + checkCPUStats(t, "", containerInfoSeed, cs.CPU) + checkMemoryStats(t, "", containerInfoSeed, containerInfo, cs.Memory) + + assert.Equal(cgroupName, cs.Name) + assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime) + + mockCadvisor.AssertExpectations(t) +} + func TestRootFsStats(t *testing.T) { const ( rootFsInfoSeed = 1000 @@ -648,6 +681,11 @@ type fakeContainerStatsProvider struct { func (p fakeContainerStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { return nil, fmt.Errorf("not implemented") } + +func (p fakeContainerStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) { + return nil, fmt.Errorf("not implemented") +} + func (p fakeContainerStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { return nil, fmt.Errorf("not implemented") }