diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 25f8683d1c1..e4a5d00c55b 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -270,26 +270,34 @@ func (p *cadvisorStatsProvider) ImageFsStats(ctx context.Context) (imageFsRet *s return imageFs, imageFs, nil } imageStats, err := p.imageService.ImageFsInfo(ctx) - if err != nil || imageStats == nil { - return nil, nil, fmt.Errorf("failed to get image stats: %v", err) + if err != nil { + return nil, nil, fmt.Errorf("failed to get image stats: %w", err) + } + if imageStats == nil || len(imageStats.ImageFilesystems) == 0 || len(imageStats.ContainerFilesystems) == 0 { + return nil, nil, fmt.Errorf("missing image stats: %+v", imageStats) } splitFileSystem := false - if imageStats.ImageFilesystems[0].FsId.Mountpoint != imageStats.ContainerFilesystems[0].FsId.Mountpoint { - klog.InfoS("Detect Split Filesystem", "ImageFilesystems", imageStats.ImageFilesystems[0], "ContainerFilesystems", imageStats.ContainerFilesystems[0]) + imageFs := imageStats.ImageFilesystems[0] + containerFs := imageStats.ContainerFilesystems[0] + if imageFs.FsId != nil && containerFs.FsId != nil && imageFs.FsId.Mountpoint != containerFs.FsId.Mountpoint { + klog.InfoS("Detect Split Filesystem", "ImageFilesystems", imageFs, "ContainerFilesystems", containerFs) splitFileSystem = true } - imageFs := imageStats.ImageFilesystems[0] var imageFsInodesUsed *uint64 if imageFsInfo.Inodes != nil && imageFsInfo.InodesFree != nil { imageFsIU := *imageFsInfo.Inodes - *imageFsInfo.InodesFree imageFsInodesUsed = &imageFsIU } + var usedBytes uint64 + if imageFs.GetUsedBytes() != nil { + usedBytes = imageFs.GetUsedBytes().GetValue() + } fsStats := &statsapi.FsStats{ Time: metav1.NewTime(imageFsInfo.Timestamp), AvailableBytes: &imageFsInfo.Available, CapacityBytes: &imageFsInfo.Capacity, - UsedBytes: &imageFs.UsedBytes.Value, + UsedBytes: &usedBytes, InodesFree: imageFsInfo.InodesFree, Inodes: imageFsInfo.Inodes, InodesUsed: imageFsInodesUsed, @@ -305,18 +313,21 @@ func (p *cadvisorStatsProvider) ImageFsStats(ctx context.Context) (imageFsRet *s return nil, nil, fmt.Errorf("failed to get container fs info: %w", err) } - containerFs := imageStats.ContainerFilesystems[0] var containerFsInodesUsed *uint64 if containerFsInfo.Inodes != nil && containerFsInfo.InodesFree != nil { containerFsIU := *containerFsInfo.Inodes - *containerFsInfo.InodesFree containerFsInodesUsed = &containerFsIU } + var usedContainerBytes uint64 + if containerFs.GetUsedBytes() != nil { + usedContainerBytes = containerFs.GetUsedBytes().GetValue() + } fsContainerStats := &statsapi.FsStats{ Time: metav1.NewTime(containerFsInfo.Timestamp), AvailableBytes: &containerFsInfo.Available, CapacityBytes: &containerFsInfo.Capacity, - UsedBytes: &containerFs.UsedBytes.Value, + UsedBytes: &usedContainerBytes, InodesFree: containerFsInfo.InodesFree, Inodes: containerFsInfo.Inodes, InodesUsed: containerFsInodesUsed, diff --git a/pkg/kubelet/stats/cadvisor_stats_provider_test.go b/pkg/kubelet/stats/cadvisor_stats_provider_test.go index bff66f7acca..4cf988f4fe1 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider_test.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider_test.go @@ -23,6 +23,7 @@ import ( cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -546,6 +547,86 @@ func TestCadvisorImagesFsStatsKubeletSeparateDiskOff(t *testing.T) { assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed) } +func TestImageFsStatsCustomResponse(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletSeparateDiskGC, true) + for desc, tc := range map[string]struct { + response *runtimeapi.ImageFsInfoResponse + callContainerFsInfo, shouldErr bool + }{ + "image stats are nil": { + shouldErr: true, + }, + "no image filesystems in image stats": { + response: &runtimeapi.ImageFsInfoResponse{ + ImageFilesystems: []*runtimeapi.FilesystemUsage{}, + ContainerFilesystems: []*runtimeapi.FilesystemUsage{{}}, + }, + shouldErr: true, + }, + "no container filesystems in image stats": { + response: &runtimeapi.ImageFsInfoResponse{ + ImageFilesystems: []*runtimeapi.FilesystemUsage{{}}, + ContainerFilesystems: []*runtimeapi.FilesystemUsage{}, + }, + shouldErr: true, + }, + "image and container filesystem identifiers are nil": { + response: &runtimeapi.ImageFsInfoResponse{ + ImageFilesystems: []*runtimeapi.FilesystemUsage{{}}, + ContainerFilesystems: []*runtimeapi.FilesystemUsage{{}}, + }, + shouldErr: false, + }, + "using different mountpoints but no used bytes set": { + response: &runtimeapi.ImageFsInfoResponse{ + ImageFilesystems: []*runtimeapi.FilesystemUsage{{ + FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: "mnt-1"}, + }}, + ContainerFilesystems: []*runtimeapi.FilesystemUsage{{ + FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: "mnt-2"}, + }}, + }, + callContainerFsInfo: true, + shouldErr: false, + }, + "using different mountpoints and set used bytes": { + response: &runtimeapi.ImageFsInfoResponse{ + ImageFilesystems: []*runtimeapi.FilesystemUsage{{ + FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: "mnt-1"}, + UsedBytes: &runtimeapi.UInt64Value{Value: 10}, + }}, + ContainerFilesystems: []*runtimeapi.FilesystemUsage{{ + FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: "mnt-2"}, + UsedBytes: &runtimeapi.UInt64Value{Value: 20}, + }}, + }, + callContainerFsInfo: true, + shouldErr: false, + }, + } { + ctx := context.Background() + mockCadvisor := cadvisortest.NewMockInterface(t) + mockRuntime := containertest.NewMockRuntime(t) + + res := getTestFsInfo(1000) + mockCadvisor.EXPECT().ImagesFsInfo().Return(res, nil) + mockRuntime.EXPECT().ImageFsInfo(ctx).Return(tc.response, nil) + if tc.callContainerFsInfo { + mockCadvisor.EXPECT().ContainerFsInfo().Return(res, nil) + } + + provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime, nil, NewFakeHostStatsProvider()) + stats, containerfs, err := provider.ImageFsStats(ctx) + if tc.shouldErr { + require.Error(t, err, desc) + assert.Nil(t, stats) + assert.Nil(t, containerfs) + } else { + assert.NoError(t, err, desc) + } + } +} + func TestCadvisorImagesFsStats(t *testing.T) { ctx := context.Background() var (