Merge pull request #108115 from haircommander/cadvisor-pod-stats

kubelet/stats: update cadvisor stats provider with new log location

pkg/kubelet/stats/cadvisor_stats_provider.go

@@ -125,7 +125,17 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 			// the user and has network stats.
 			podStats.Network = cadvisorInfoToNetworkStats(&cinfo)
 		} else {
-			podStats.Containers = append(podStats.Containers, *cadvisorInfoToContainerStats(containerName, &cinfo, &rootFsInfo, &imageFsInfo))
+			containerStat := cadvisorInfoToContainerStats(containerName, &cinfo, &rootFsInfo, &imageFsInfo)
+			// NOTE: This doesn't support the old pod log path, `/var/log/pods/UID`. For containers
+			// using old log path, they will be populated by cadvisorInfoToContainerStats.
+			podUID := types.UID(podStats.PodRef.UID)
+			logs, err := p.hostStatsProvider.getPodContainerLogStats(podStats.PodRef.Namespace, podStats.PodRef.Name, podUID, containerName, &rootFsInfo)
+			if err != nil {
+				klog.ErrorS(err, "Unable to fetch container log stats", "containerName", containerName)
+			} else {
+				containerStat.Logs = logs
+			}
+			podStats.Containers = append(podStats.Containers, *containerStat)
 		}
 	}
 

pkg/kubelet/stats/cadvisor_stats_provider_test.go

@@ -30,9 +30,12 @@ import (
 	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
 	"k8s.io/kubernetes/pkg/kubelet/leaky"
 	serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
 	statustest "k8s.io/kubernetes/pkg/kubelet/status/testing"
+	"k8s.io/kubernetes/pkg/volume"
 )
 
 func TestFilterTerminatedContainerInfoAndAssembleByPodCgroupKey(t *testing.T) {
@@ -270,7 +273,7 @@ func TestCadvisorListPodStats(t *testing.T) {
 
 	assert.EqualValues(t, p0Time.Unix(), ps.StartTime.Time.Unix())
 	checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)
-	checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, []int{seedEphemeralVolume1, seedEphemeralVolume2}, ps.EphemeralStorage)
+	checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, []int{seedEphemeralVolume1, seedEphemeralVolume2}, nil, ps.EphemeralStorage)
 	if ps.CPU != nil {
 		checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
 	}
@@ -510,3 +513,106 @@ func TestCadvisorImagesFsStats(t *testing.T) {
 	assert.Equal(imageFsInfo.Inodes, stats.Inodes)
 	assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed)
 }
+
+func TestCadvisorListPodStatsWhenContainerLogFound(t *testing.T) {
+	const (
+		namespace0 = "test0"
+	)
+	const (
+		seedRoot           = 0
+		seedRuntime        = 100
+		seedKubelet        = 200
+		seedMisc           = 300
+		seedPod0Infra      = 1000
+		seedPod0Container0 = 0
+		seedPod0Container1 = 0
+	)
+	const (
+		pName0 = "pod0"
+	)
+	const (
+		cName00 = "c0"
+		cName01 = "c1"
+	)
+	const (
+		rootfsCapacity    = uint64(10000000)
+		rootfsAvailable   = uint64(5000000)
+		rootfsInodesFree  = uint64(1000)
+		rootfsInodes      = uint64(2000)
+		imagefsCapacity   = uint64(20000000)
+		imagefsAvailable  = uint64(8000000)
+		imagefsInodesFree = uint64(2000)
+		imagefsInodes     = uint64(4000)
+	)
+
+	prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0}
+	infos := map[string]cadvisorapiv2.ContainerInfo{
+		"/":              getTestContainerInfo(seedRoot, "", "", ""),
+		"/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""),
+		"/kubelet":       getTestContainerInfo(seedKubelet, "", "", ""),
+		"/system":        getTestContainerInfo(seedMisc, "", "", ""),
+		// Pod0 - Namespace0
+		"/pod0-i":  getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
+		"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00),
+		"/pod0-c1": getTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01),
+	}
+
+	containerLogStats0 := makeFakeLogStats(0)
+	containerLogStats1 := makeFakeLogStats(0)
+	fakeStats := map[string]*volume.Metrics{
+		kuberuntime.BuildContainerLogsDirectory(prf0.Namespace, prf0.Name, types.UID(prf0.UID), cName00): containerLogStats0,
+		kuberuntime.BuildContainerLogsDirectory(prf0.Namespace, prf0.Name, types.UID(prf0.UID), cName01): containerLogStats1,
+	}
+	fakeStatsSlice := []*volume.Metrics{containerLogStats0, containerLogStats1}
+	fakeOS := &kubecontainertest.FakeOS{}
+
+	freeRootfsInodes := rootfsInodesFree
+	totalRootfsInodes := rootfsInodes
+	rootfs := cadvisorapiv2.FsInfo{
+		Capacity:   rootfsCapacity,
+		Available:  rootfsAvailable,
+		InodesFree: &freeRootfsInodes,
+		Inodes:     &totalRootfsInodes,
+	}
+
+	freeImagefsInodes := imagefsInodesFree
+	totalImagefsInodes := imagefsInodes
+	imagefs := cadvisorapiv2.FsInfo{
+		Capacity:   imagefsCapacity,
+		Available:  imagefsAvailable,
+		InodesFree: &freeImagefsInodes,
+		Inodes:     &totalImagefsInodes,
+	}
+
+	options := cadvisorapiv2.RequestOptions{
+		IdType:    cadvisorapiv2.TypeName,
+		Count:     2,
+		Recursive: true,
+	}
+
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	mockCadvisor := cadvisortest.NewMockInterface(mockCtrl)
+	mockCadvisor.EXPECT().ContainerInfoV2("/", options).Return(infos, nil)
+	mockCadvisor.EXPECT().RootFsInfo().Return(rootfs, nil)
+	mockCadvisor.EXPECT().ImagesFsInfo().Return(imagefs, nil)
+
+	mockRuntime := containertest.NewMockRuntime(mockCtrl)
+	mockRuntime.EXPECT().ImageStats().Return(&kubecontainer.ImageStats{TotalStorageBytes: 123}, nil).AnyTimes()
+
+	volumeStats := serverstats.PodVolumeStats{}
+	p0Time := metav1.Now()
+	mockStatus := statustest.NewMockPodStatusProvider(mockCtrl)
+	mockStatus.EXPECT().GetPodStatus(types.UID("UID"+pName0)).Return(v1.PodStatus{StartTime: &p0Time}, true)
+
+	resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
+
+	p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime, mockStatus, NewFakeHostStatsProviderWithData(fakeStats, fakeOS))
+	pods, err := p.ListPodStats()
+	assert.NoError(t, err)
+
+	assert.Equal(t, 1, len(pods))
+	// Validate Pod0 Results
+	checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, nil, fakeStatsSlice, pods[0].EphemeralStorage)
+}

pkg/kubelet/stats/helper.go

@@ -94,6 +94,9 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
 	result.CPU = cpu
 	result.Memory = memory
 
+	// NOTE: if they can be found, log stats will be overwritten
+	// by the caller, as it knows more information about the pod,
+	// which is needed to determine log size.
 	if rootFs != nil {
 		// The container logs live on the node rootfs device
 		result.Logs = buildLogsStats(cstat, rootFs)

pkg/kubelet/stats/provider_test.go

@@ -37,6 +37,7 @@ import (
 	kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
 	serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
+	"k8s.io/kubernetes/pkg/volume"
 )
 
 const (
@@ -699,16 +700,27 @@ func checkFsStats(t *testing.T, label string, seed int, stats *statsapi.FsStats)
 	assert.EqualValues(t, seed+offsetFsInodesFree, *stats.InodesFree, label+".InodesFree")
 }
 
-func checkEphemeralStats(t *testing.T, label string, containerSeeds []int, volumeSeeds []int, stats *statsapi.FsStats) {
+func checkEphemeralStats(t *testing.T, label string, containerSeeds []int, volumeSeeds []int, containerLogStats []*volume.Metrics, stats *statsapi.FsStats) {
 	var usedBytes, inodeUsage int
 	for _, cseed := range containerSeeds {
-		usedBytes = usedBytes + cseed + offsetFsTotalUsageBytes
+		usedBytes += cseed + offsetFsBaseUsageBytes
 		inodeUsage += cseed + offsetFsInodeUsage
+		// If containerLogStats is nil, then the log stats calculated from cAdvisor
+		// information is used. Since it's Total - Base, and these values are
+		// set to the offset, we can use the calculated difference in the offset
+		// to account for this.
+		if containerLogStats == nil {
+			usedBytes += offsetFsTotalUsageBytes - offsetFsBaseUsageBytes
+		}
 	}
 	for _, vseed := range volumeSeeds {
-		usedBytes = usedBytes + vseed + offsetFsUsage
+		usedBytes += vseed + offsetFsUsage
 		inodeUsage += vseed + offsetFsInodeUsage
 	}
+	for _, logStats := range containerLogStats {
+		usedBytes += int(logStats.Used.Value())
+		inodeUsage += int(logStats.InodesUsed.Value())
+	}
 	assert.EqualValues(t, usedBytes, int(*stats.UsedBytes), label+".UsedBytes")
 	assert.EqualValues(t, inodeUsage, int(*stats.InodesUsed), label+".InodesUsed")
 }