Merge pull request #59906 from abhi/log_stats

Automatic merge from submit-queue (batch tested with PRs 54191, 59374, 59824, 55032, 59906). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adding per container stats for CRI runtimes

**What this PR does / why we need it**

This commit aims to collect per container log stats. The change was proposed as a part of #55905. The change includes change the log path from /var/pod/<pod uid>/containername_attempt.log to /var/pod/<pod uid>/containername/containername_attempt.log. The logs are collected by reusing volume package to collect metrics from the log path.
Fixes #55905

**Special notes for your reviewer:**
cc @Random-Liu

**Release note:**

```
Adding container log stats for CRI runtimes.
```

pkg/kubelet/kubelet.go

@@ -695,7 +695,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 				klet.podManager,
 				klet.runtimeCache,
 				runtimeService,
-				imageService)
+				imageService,
+				stats.NewLogMetricsService())
 		}
 	} else {
 		// rkt uses the legacy, non-CRI, integration. Configure it the old way.

pkg/kubelet/kuberuntime/helpers.go

@@ -207,7 +207,7 @@ func getStableKey(pod *v1.Pod, container *v1.Container) string {
 
 // buildContainerLogsPath builds log path for container relative to pod logs directory.
 func buildContainerLogsPath(containerName string, restartCount int) string {
-	return fmt.Sprintf("%s_%d.log", containerName, restartCount)
+	return filepath.Join(containerName, fmt.Sprintf("%d.log", restartCount))
 }
 
 // buildFullContainerLogsPath builds absolute log path for container.
@@ -215,6 +215,11 @@ func buildFullContainerLogsPath(podUID types.UID, containerName string, restartC
 	return filepath.Join(buildPodLogsDirectory(podUID), buildContainerLogsPath(containerName, restartCount))
 }
 
+// BuildContainerLogsDirectory builds absolute log directory path for a container in pod.
+func BuildContainerLogsDirectory(podUID types.UID, containerName string) string {
+	return filepath.Join(buildPodLogsDirectory(podUID), containerName)
+}
+
 // buildPodLogsDirectory builds absolute log directory path for a pod sandbox.
 func buildPodLogsDirectory(podUID types.UID) string {
 	return filepath.Join(podLogsRootDirectory, string(podUID))

pkg/kubelet/kuberuntime/kuberuntime_container.go

@@ -190,6 +190,11 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai
 	}
 
 	command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
+	logDir := BuildContainerLogsDirectory(kubetypes.UID(pod.UID), container.Name)
+	err = m.osInterface.MkdirAll(logDir, 0755)
+	if err != nil {
+		return nil, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err)
+	}
 	containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
 	restartCountUint32 := uint32(restartCount)
 	config := &runtimeapi.ContainerConfig{

pkg/kubelet/kuberuntime/kuberuntime_container_test.go

@@ -61,7 +61,7 @@ func TestRemoveContainer(t *testing.T) {
 	err = m.removeContainer(containerId)
 	assert.NoError(t, err)
 	// Verify container log is removed
-	expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "12345678", "foo_0.log")
+	expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "12345678", "foo", "0.log")
 	expectedContainerLogSymlink := legacyLogSymlink(containerId, "foo", "bar", "new")
 	assert.Equal(t, fakeOS.Removes, []string{expectedContainerLogPath, expectedContainerLogSymlink})
 	// Verify container is removed

pkg/kubelet/stats/BUILD

@@ -6,6 +6,7 @@ go_library(
         "cadvisor_stats_provider.go",
         "cri_stats_provider.go",
         "helper.go",
+        "log_metrics_provider.go",
         "stats_provider.go",
     ] + select({
         "@io_bazel_rules_go//go/platform:android": [
@@ -52,11 +53,13 @@ go_library(
         "//pkg/kubelet/cadvisor:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/kuberuntime:go_default_library",
         "//pkg/kubelet/leaky:go_default_library",
         "//pkg/kubelet/network:go_default_library",
         "//pkg/kubelet/pod:go_default_library",
         "//pkg/kubelet/server/stats:go_default_library",
         "//pkg/kubelet/types:go_default_library",
+        "//pkg/volume:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/golang/protobuf/proto:go_default_library",
         "//vendor/github.com/google/cadvisor/fs:go_default_library",
@@ -87,6 +90,7 @@ go_test(
         "cadvisor_stats_provider_test.go",
         "cri_stats_provider_test.go",
         "helper_test.go",
+        "log_metrics_provider_test.go",
         "stats_provider_test.go",
     ],
     embed = [":go_default_library"],
@@ -97,16 +101,19 @@ go_test(
         "//pkg/kubelet/cadvisor/testing:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/container/testing:go_default_library",
+        "//pkg/kubelet/kuberuntime:go_default_library",
         "//pkg/kubelet/leaky:go_default_library",
         "//pkg/kubelet/pod/testing:go_default_library",
         "//pkg/kubelet/server/stats:go_default_library",
         "//pkg/kubelet/types:go_default_library",
+        "//pkg/volume:go_default_library",
         "//vendor/github.com/google/cadvisor/fs:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
         "//vendor/github.com/google/gofuzz:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
     ],

pkg/kubelet/stats/cri_stats_provider.go

@@ -35,6 +35,7 @@ import (
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
+	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
 	"k8s.io/kubernetes/pkg/kubelet/server/stats"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
@@ -53,6 +54,8 @@ type criStatsProvider struct {
 	runtimeService internalapi.RuntimeService
 	// imageService is used to get the stats of the image filesystem.
 	imageService internalapi.ImageManagerService
+	// logMetrics provides the metrics for container logs
+	logMetricsService LogMetricsService
 }
 
 // newCRIStatsProvider returns a containerStatsProvider implementation that
@@ -62,12 +65,14 @@ func newCRIStatsProvider(
 	resourceAnalyzer stats.ResourceAnalyzer,
 	runtimeService internalapi.RuntimeService,
 	imageService internalapi.ImageManagerService,
+	logMetricsService LogMetricsService,
 ) containerStatsProvider {
 	return &criStatsProvider{
-		cadvisor:         cadvisor,
+		cadvisor:          cadvisor,
-		resourceAnalyzer: resourceAnalyzer,
+		resourceAnalyzer:  resourceAnalyzer,
-		runtimeService:   runtimeService,
+		runtimeService:    runtimeService,
-		imageService:     imageService,
+		imageService:      imageService,
+		logMetricsService: logMetricsService,
 	}
 }
 
@@ -94,7 +99,6 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 	for _, s := range podSandboxes {
 		podSandboxMap[s.Id] = s
 	}
-
 	// fsIDtoInfo is a map from filesystem id to its stats. This will be used
 	// as a cache to avoid querying cAdvisor for the filesystem stats with the
 	// same filesystem id many times.
@@ -149,7 +153,7 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 			}
 			sandboxIDToPodStats[podSandboxID] = ps
 		}
-		cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo)
+		cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid())
 		// If cadvisor stats is available for the container, use it to populate
 		// container stats
 		caStats, caFound := caInfos[containerID]
@@ -277,6 +281,7 @@ func (p *criStatsProvider) makeContainerStats(
 	container *runtimeapi.Container,
 	rootFsInfo *cadvisorapiv2.FsInfo,
 	fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo,
+	uid string,
 ) *statsapi.ContainerStats {
 	result := &statsapi.ContainerStats{
 		Name: stats.Attributes.Metadata.Name,
@@ -291,17 +296,6 @@ func (p *criStatsProvider) makeContainerStats(
 			RSSBytes: proto.Uint64(0),
 		},
 		Rootfs: &statsapi.FsStats{},
-		Logs: &statsapi.FsStats{
-			Time:           metav1.NewTime(rootFsInfo.Timestamp),
-			AvailableBytes: &rootFsInfo.Available,
-			CapacityBytes:  &rootFsInfo.Capacity,
-			InodesFree:     rootFsInfo.InodesFree,
-			Inodes:         rootFsInfo.Inodes,
-			// UsedBytes and InodesUsed are unavailable from CRI stats.
-			//
-			// TODO(yguo0905): Get this information from kubelet and
-			// populate the two fields here.
-		},
 		// UserDefinedMetrics is not supported by CRI.
 	}
 	if stats.Cpu != nil {
@@ -343,7 +337,8 @@ func (p *criStatsProvider) makeContainerStats(
 			result.Rootfs.Inodes = imageFsInfo.Inodes
 		}
 	}
-
+	containerLogPath := kuberuntime.BuildContainerLogsDirectory(types.UID(uid), container.GetMetadata().GetName())
+	result.Logs = p.getContainerLogStats(containerLogPath, rootFsInfo)
 	return result
 }
 
@@ -423,3 +418,25 @@ func getCRICadvisorStats(ca cadvisor.Interface) (map[string]cadvisorapiv2.Contai
 	}
 	return stats, nil
 }
+
+// TODO Cache the metrics in container log manager
+func (p *criStatsProvider) getContainerLogStats(path string, rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.FsStats {
+	m := p.logMetricsService.createLogMetricsProvider(path)
+	logMetrics, err := m.GetMetrics()
+	if err != nil {
+		glog.Errorf("Unable to fetch container log stats for path %s: %v ", path, err)
+		return nil
+	}
+	result := &statsapi.FsStats{
+		Time:           metav1.NewTime(rootFsInfo.Timestamp),
+		AvailableBytes: &rootFsInfo.Available,
+		CapacityBytes:  &rootFsInfo.Capacity,
+		InodesFree:     rootFsInfo.InodesFree,
+		Inodes:         rootFsInfo.Inodes,
+	}
+	usedbytes := uint64(logMetrics.Used.Value())
+	result.UsedBytes = &usedbytes
+	inodesUsed := uint64(logMetrics.InodesUsed.Value())
+	result.InodesUsed = &inodesUsed
+	return result
+}

pkg/kubelet/stats/cri_stats_provider_test.go

@@ -24,15 +24,24 @@ import (
 	cadvisorfs "github.com/google/cadvisor/fs"
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
 	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
 	kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
 	"k8s.io/kubernetes/pkg/kubelet/leaky"
 	kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
 	serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+const (
+	offsetInodeUsage = iota
+	offsetUsage
 )
 
 func TestCRIListPodStats(t *testing.T) {
@@ -68,21 +77,25 @@ func TestCRIListPodStats(t *testing.T) {
 		imageFsInfo       = getTestFsInfo(2000)
 		rootFsInfo        = getTestFsInfo(1000)
 
-		sandbox0        = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
+		sandbox0           = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
-		container0      = makeFakeContainer(sandbox0, cName0, 0, false)
+		container0         = makeFakeContainer(sandbox0, cName0, 0, false)
-		containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint)
+		containerStats0    = makeFakeContainerStats(container0, imageFsMountpoint)
-		container1      = makeFakeContainer(sandbox0, cName1, 0, false)
+		containerLogStats0 = makeFakeLogStats(1000)
-		containerStats1 = makeFakeContainerStats(container1, unknownMountpoint)
+		container1         = makeFakeContainer(sandbox0, cName1, 0, false)
+		containerStats1    = makeFakeContainerStats(container1, unknownMountpoint)
+		containerLogStats1 = makeFakeLogStats(2000)
 
-		sandbox1        = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
+		sandbox1           = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
-		container2      = makeFakeContainer(sandbox1, cName2, 0, false)
+		container2         = makeFakeContainer(sandbox1, cName2, 0, false)
-		containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint)
+		containerStats2    = makeFakeContainerStats(container2, imageFsMountpoint)
+		containerLogStats2 = makeFakeLogStats(3000)
 
-		sandbox2        = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
+		sandbox2           = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
-		container3      = makeFakeContainer(sandbox2, cName3, 0, true)
+		container3         = makeFakeContainer(sandbox2, cName3, 0, true)
-		containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint)
+		containerStats3    = makeFakeContainerStats(container3, imageFsMountpoint)
-		container4      = makeFakeContainer(sandbox2, cName3, 1, false)
+		container4         = makeFakeContainer(sandbox2, cName3, 1, false)
-		containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
+		containerStats4    = makeFakeContainerStats(container4, imageFsMountpoint)
+		containerLogStats4 = makeFakeLogStats(4000)
 	)
 
 	var (
@@ -135,13 +148,23 @@ func TestCRIListPodStats(t *testing.T) {
 		PersistentVolumes: persistentVolumes,
 	}
 
+	fakeLogStats := map[string]*volume.Metrics{
+		kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName0): containerLogStats0,
+		kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName1): containerLogStats1,
+		kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox1-uid"), cName2): containerLogStats2,
+		kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox2-uid"), cName3): containerLogStats4,
+	}
+	fakeLogStatsProvider := NewFakeLogMetricsService(fakeLogStats)
+
 	provider := NewCRIStatsProvider(
 		mockCadvisor,
 		resourceAnalyzer,
 		mockPodManager,
 		mockRuntimeCache,
 		fakeRuntimeService,
-		fakeImageService)
+		fakeImageService,
+		fakeLogStatsProvider,
+	)
 
 	stats, err := provider.ListPodStats()
 	assert := assert.New(t)
@@ -157,7 +180,8 @@ func TestCRIListPodStats(t *testing.T) {
 	assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
 	assert.Equal(2, len(p0.Containers))
 
-	checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1})
+	checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1},
+		[]*volume.Metrics{containerLogStats0, containerLogStats1})
 
 	containerStatsMap := make(map[string]statsapi.ContainerStats)
 	for _, s := range p0.Containers {
@@ -168,32 +192,32 @@ func TestCRIListPodStats(t *testing.T) {
 	assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
 	checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
 	checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
-	checkCRILogsStats(assert, c0, &rootFsInfo)
+	checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0)
 	c1 := containerStatsMap[cName1]
 	assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
 	checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
 	checkCRIRootfsStats(assert, c1, containerStats1, nil)
-	checkCRILogsStats(assert, c1, &rootFsInfo)
+	checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1)
 	checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
 
 	p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
 	assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
 	assert.Equal(1, len(p1.Containers))
 
-	checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2})
+	checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2}, []*volume.Metrics{containerLogStats2})
 	c2 := p1.Containers[0]
 	assert.Equal(cName2, c2.Name)
 	assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
 	checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
 	checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
-	checkCRILogsStats(assert, c2, &rootFsInfo)
+	checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2)
 	checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
 
 	p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
 	assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
 	assert.Equal(1, len(p2.Containers))
 
-	checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4})
+	checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4}, []*volume.Metrics{containerLogStats4})
 
 	c3 := p2.Containers[0]
 	assert.Equal(cName3, c3.Name)
@@ -201,7 +225,7 @@ func TestCRIListPodStats(t *testing.T) {
 	checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
 	checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
 
-	checkCRILogsStats(assert, c3, &rootFsInfo)
+	checkCRILogsStats(assert, c3, &rootFsInfo, containerLogStats4)
 	checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
 
 	mockCadvisor.AssertExpectations(t)
@@ -214,12 +238,13 @@ func TestCRIImagesFsStats(t *testing.T) {
 		imageFsUsage      = makeFakeImageFsUsage(imageFsMountpoint)
 	)
 	var (
-		mockCadvisor       = new(cadvisortest.Mock)
+		mockCadvisor         = new(cadvisortest.Mock)
-		mockRuntimeCache   = new(kubecontainertest.MockRuntimeCache)
+		mockRuntimeCache     = new(kubecontainertest.MockRuntimeCache)
-		mockPodManager     = new(kubepodtest.MockManager)
+		mockPodManager       = new(kubepodtest.MockManager)
-		resourceAnalyzer   = new(fakeResourceAnalyzer)
+		resourceAnalyzer     = new(fakeResourceAnalyzer)
-		fakeRuntimeService = critest.NewFakeRuntimeService()
+		fakeRuntimeService   = critest.NewFakeRuntimeService()
-		fakeImageService   = critest.NewFakeImageService()
+		fakeImageService     = critest.NewFakeImageService()
+		fakeLogStatsProvider = NewFakeLogMetricsService(nil)
 	)
 
 	mockCadvisor.On("GetDirFsInfo", imageFsMountpoint).Return(imageFsInfo, nil)
@@ -233,7 +258,9 @@ func TestCRIImagesFsStats(t *testing.T) {
 		mockPodManager,
 		mockRuntimeCache,
 		fakeRuntimeService,
-		fakeImageService)
+		fakeImageService,
+		fakeLogStatsProvider,
+	)
 
 	stats, err := provider.ImageFsStats()
 	assert := assert.New(t)
@@ -385,17 +412,21 @@ func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerSta
 	assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed)
 }
 
-func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo) {
+func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo, logStats *volume.Metrics) {
 	assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time)
 	assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes)
 	assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes)
 	assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree)
 	assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes)
-	assert.Nil(actual.Logs.UsedBytes)
+	assert.Equal(uint64(logStats.Used.Value()), *actual.Logs.UsedBytes)
-	assert.Nil(actual.Logs.InodesUsed)
+	assert.Equal(uint64(logStats.InodesUsed.Value()), *actual.Logs.InodesUsed)
 }
 
-func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodStats, volumes []statsapi.VolumeStats, containers []*runtimeapi.ContainerStats) {
+func checkEphemeralStorageStats(assert *assert.Assertions,
+	actual statsapi.PodStats,
+	volumes []statsapi.VolumeStats,
+	containers []*runtimeapi.ContainerStats,
+	containerLogStats []*volume.Metrics) {
 	var totalUsed, inodesUsed uint64
 	for _, container := range containers {
 		totalUsed = totalUsed + container.WritableLayer.UsedBytes.Value
@@ -406,8 +437,13 @@ func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodSt
 		totalUsed = totalUsed + *volume.FsStats.UsedBytes
 		inodesUsed = inodesUsed + *volume.FsStats.InodesUsed
 	}
-	assert.Equal(int(*actual.EphemeralStorage.UsedBytes), int(totalUsed))
+
-	assert.Equal(int(*actual.EphemeralStorage.InodesUsed), int(inodesUsed))
+	for _, logStats := range containerLogStats {
+		totalUsed = totalUsed + uint64(logStats.Used.Value())
+	}
+
+	assert.Equal(int(totalUsed), int(*actual.EphemeralStorage.UsedBytes))
+	assert.Equal(int(inodesUsed), int(*actual.EphemeralStorage.InodesUsed))
 }
 
 func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkStats, expected *cadvisorapiv2.NetworkStats) {
@@ -416,3 +452,10 @@ func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkSta
 	assert.Equal(expected.Interfaces[0].TxBytes, *actual.TxBytes)
 	assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors)
 }
+
+func makeFakeLogStats(seed int) *volume.Metrics {
+	m := &volume.Metrics{}
+	m.Used = resource.NewQuantity(int64(seed+offsetUsage), resource.BinarySI)
+	m.InodesUsed = resource.NewQuantity(int64(seed+offsetInodeUsage), resource.BinarySI)
+	return m
+}

pkg/kubelet/stats/log_metrics_provider.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stats
+
+import (
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+type LogMetricsService interface {
+	createLogMetricsProvider(path string) volume.MetricsProvider
+}
+
+type logMetrics struct{}
+
+func NewLogMetricsService() LogMetricsService {
+	return logMetrics{}
+}
+
+func (l logMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
+	return volume.NewMetricsDu(path)
+}

pkg/kubelet/stats/log_metrics_provider_test.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stats
+
+import (
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+type fakeLogMetrics struct {
+	fakeStats map[string]*volume.Metrics
+}
+
+func NewFakeLogMetricsService(stats map[string]*volume.Metrics) LogMetricsService {
+	return &fakeLogMetrics{fakeStats: stats}
+}
+
+func (l *fakeLogMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
+	return NewFakeMetricsDu(path, l.fakeStats[path])
+}
+
+type fakeMetricsDu struct {
+	fakeStats *volume.Metrics
+}
+
+func NewFakeMetricsDu(path string, stats *volume.Metrics) volume.MetricsProvider {
+	return &fakeMetricsDu{fakeStats: stats}
+}
+
+func (f *fakeMetricsDu) GetMetrics() (*volume.Metrics, error) {
+	return f.fakeStats, nil
+}

pkg/kubelet/stats/stats_provider.go

@@ -39,8 +39,9 @@ func NewCRIStatsProvider(
 	runtimeCache kubecontainer.RuntimeCache,
 	runtimeService internalapi.RuntimeService,
 	imageService internalapi.ImageManagerService,
+	logMetricsService LogMetricsService,
 ) *StatsProvider {
-	return newStatsProvider(cadvisor, podManager, runtimeCache, newCRIStatsProvider(cadvisor, resourceAnalyzer, runtimeService, imageService))
+	return newStatsProvider(cadvisor, podManager, runtimeCache, newCRIStatsProvider(cadvisor, resourceAnalyzer, runtimeService, imageService, logMetricsService))
 }
 
 // NewCadvisorStatsProvider returns a containerStatsProvider that provides both