From 6649d38c96a34becea4fd6ae5031df9d19e0feed Mon Sep 17 00:00:00 2001 From: abhi Date: Tue, 20 Feb 2018 19:49:51 -0800 Subject: [PATCH 1/2] Adding per container stats for CRI runtimes This commit aims to collect per container log stats. The change was proposed as a part of #55905. The change includes change of the log path from /var/pod//containername_attempt.log to /var/pod//containername/containername_attempt.log. The logs are collected by reusing volume package to collect metrics from the log path. Signed-off-by: abhi --- pkg/kubelet/kubelet.go | 3 +- pkg/kubelet/kuberuntime/helpers.go | 7 ++- .../kuberuntime/kuberuntime_container.go | 8 ++- pkg/kubelet/stats/BUILD | 7 +++ pkg/kubelet/stats/cri_stats_provider.go | 53 ++++++++++++------- pkg/kubelet/stats/log_metrics_provider.go | 35 ++++++++++++ pkg/kubelet/stats/stats_provider.go | 3 +- 7 files changed, 93 insertions(+), 23 deletions(-) create mode 100644 pkg/kubelet/stats/log_metrics_provider.go diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index c1e1aa1e611..3198f968b17 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -694,7 +694,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.podManager, klet.runtimeCache, runtimeService, - imageService) + imageService, + stats.NewLogMetricsService()) } } else { // rkt uses the legacy, non-CRI, integration. Configure it the old way. diff --git a/pkg/kubelet/kuberuntime/helpers.go b/pkg/kubelet/kuberuntime/helpers.go index ef087171b32..34bcb130543 100644 --- a/pkg/kubelet/kuberuntime/helpers.go +++ b/pkg/kubelet/kuberuntime/helpers.go @@ -207,7 +207,7 @@ func getStableKey(pod *v1.Pod, container *v1.Container) string { // buildContainerLogsPath builds log path for container relative to pod logs directory. func buildContainerLogsPath(containerName string, restartCount int) string { - return fmt.Sprintf("%s_%d.log", containerName, restartCount) + return filepath.Join(containerName, fmt.Sprintf("%d.log", restartCount)) } // buildFullContainerLogsPath builds absolute log path for container. @@ -215,6 +215,11 @@ func buildFullContainerLogsPath(podUID types.UID, containerName string, restartC return filepath.Join(buildPodLogsDirectory(podUID), buildContainerLogsPath(containerName, restartCount)) } +// BuildContainerLogsDirectory builds absolute log directory path for a container in pod. +func BuildContainerLogsDirectory(podUID types.UID, containerName string) string { + return filepath.Join(buildPodLogsDirectory(podUID), containerName) +} + // buildPodLogsDirectory builds absolute log directory path for a pod sandbox. func buildPodLogsDirectory(podUID types.UID) string { return filepath.Join(podLogsRootDirectory, string(podUID)) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 14886b4390c..424a4e3a70a 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -190,6 +190,11 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai } command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs) + logDir := BuildContainerLogsDirectory(kubetypes.UID(pod.UID), container.Name) + err = m.osInterface.MkdirAll(logDir, 0755) + if err != nil { + return nil, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err) + } containerLogsPath := buildContainerLogsPath(container.Name, restartCount) restartCountUint32 := uint32(restartCount) config := &runtimeapi.ContainerConfig{ @@ -840,8 +845,7 @@ func (m *kubeGenericRuntimeManager) removeContainerLog(containerID string) error return fmt.Errorf("failed to get container status %q: %v", containerID, err) } labeledInfo := getContainerInfoFromLabels(status.Labels) - annotatedInfo := getContainerInfoFromAnnotations(status.Annotations) - path := buildFullContainerLogsPath(labeledInfo.PodUID, labeledInfo.ContainerName, annotatedInfo.RestartCount) + path := BuildContainerLogsDirectory(labeledInfo.PodUID, labeledInfo.ContainerName) if err := m.osInterface.Remove(path); err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to remove container %q log %q: %v", containerID, path, err) } diff --git a/pkg/kubelet/stats/BUILD b/pkg/kubelet/stats/BUILD index 68463fc0a6a..b66f834665c 100644 --- a/pkg/kubelet/stats/BUILD +++ b/pkg/kubelet/stats/BUILD @@ -6,6 +6,7 @@ go_library( "cadvisor_stats_provider.go", "cri_stats_provider.go", "helper.go", + "log_metrics_provider.go", "stats_provider.go", ], importpath = "k8s.io/kubernetes/pkg/kubelet/stats", @@ -17,11 +18,13 @@ go_library( "//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", + "//pkg/kubelet/kuberuntime:go_default_library", "//pkg/kubelet/leaky:go_default_library", "//pkg/kubelet/network:go_default_library", "//pkg/kubelet/pod:go_default_library", "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/types:go_default_library", + "//pkg/volume:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/protobuf/proto:go_default_library", "//vendor/github.com/google/cadvisor/fs:go_default_library", @@ -52,6 +55,7 @@ go_test( "cadvisor_stats_provider_test.go", "cri_stats_provider_test.go", "helper_test.go", + "log_metrics_provider_test.go", "stats_provider_test.go", ], embed = [":go_default_library"], @@ -63,16 +67,19 @@ go_test( "//pkg/kubelet/cadvisor/testing:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/container/testing:go_default_library", + "//pkg/kubelet/kuberuntime:go_default_library", "//pkg/kubelet/leaky:go_default_library", "//pkg/kubelet/pod/testing:go_default_library", "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/types:go_default_library", + "//pkg/volume:go_default_library", "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", ], diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index 42f732a7b84..64955edb470 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -35,6 +35,7 @@ import ( runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/kuberuntime" "k8s.io/kubernetes/pkg/kubelet/server/stats" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -53,6 +54,8 @@ type criStatsProvider struct { runtimeService internalapi.RuntimeService // imageService is used to get the stats of the image filesystem. imageService internalapi.ImageManagerService + // logMetrics provides the metrics for container logs + logMetricsService LogMetricsService } // newCRIStatsProvider returns a containerStatsProvider implementation that @@ -62,12 +65,14 @@ func newCRIStatsProvider( resourceAnalyzer stats.ResourceAnalyzer, runtimeService internalapi.RuntimeService, imageService internalapi.ImageManagerService, + logMetricsService LogMetricsService, ) containerStatsProvider { return &criStatsProvider{ - cadvisor: cadvisor, - resourceAnalyzer: resourceAnalyzer, - runtimeService: runtimeService, - imageService: imageService, + cadvisor: cadvisor, + resourceAnalyzer: resourceAnalyzer, + runtimeService: runtimeService, + imageService: imageService, + logMetricsService: logMetricsService, } } @@ -94,7 +99,6 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { for _, s := range podSandboxes { podSandboxMap[s.Id] = s } - // fsIDtoInfo is a map from filesystem id to its stats. This will be used // as a cache to avoid querying cAdvisor for the filesystem stats with the // same filesystem id many times. @@ -149,7 +153,7 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { } sandboxIDToPodStats[podSandboxID] = ps } - cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo) + cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid()) // If cadvisor stats is available for the container, use it to populate // container stats caStats, caFound := caInfos[containerID] @@ -277,6 +281,7 @@ func (p *criStatsProvider) makeContainerStats( container *runtimeapi.Container, rootFsInfo *cadvisorapiv2.FsInfo, fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo, + uid string, ) *statsapi.ContainerStats { result := &statsapi.ContainerStats{ Name: stats.Attributes.Metadata.Name, @@ -291,17 +296,6 @@ func (p *criStatsProvider) makeContainerStats( RSSBytes: proto.Uint64(0), }, Rootfs: &statsapi.FsStats{}, - Logs: &statsapi.FsStats{ - Time: metav1.NewTime(rootFsInfo.Timestamp), - AvailableBytes: &rootFsInfo.Available, - CapacityBytes: &rootFsInfo.Capacity, - InodesFree: rootFsInfo.InodesFree, - Inodes: rootFsInfo.Inodes, - // UsedBytes and InodesUsed are unavailable from CRI stats. - // - // TODO(yguo0905): Get this information from kubelet and - // populate the two fields here. - }, // UserDefinedMetrics is not supported by CRI. } if stats.Cpu != nil { @@ -343,7 +337,8 @@ func (p *criStatsProvider) makeContainerStats( result.Rootfs.Inodes = imageFsInfo.Inodes } } - + containerLogPath := kuberuntime.BuildContainerLogsDirectory(types.UID(uid), container.GetMetadata().GetName()) + result.Logs = p.getContainerLogStats(containerLogPath, rootFsInfo) return result } @@ -423,3 +418,25 @@ func getCRICadvisorStats(ca cadvisor.Interface) (map[string]cadvisorapiv2.Contai } return stats, nil } + +// TODO Cache the metrics in container log manager +func (p *criStatsProvider) getContainerLogStats(path string, rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.FsStats { + m := p.logMetricsService.createLogMetricsProvider(path) + logMetrics, err := m.GetMetrics() + if err != nil { + glog.Errorf("Unable to fetch container log stats for path %s: %v ", path, err) + return nil + } + result := &statsapi.FsStats{ + Time: metav1.NewTime(rootFsInfo.Timestamp), + AvailableBytes: &rootFsInfo.Available, + CapacityBytes: &rootFsInfo.Capacity, + InodesFree: rootFsInfo.InodesFree, + Inodes: rootFsInfo.Inodes, + } + usedbytes := uint64(logMetrics.Used.Value()) + result.UsedBytes = &usedbytes + inodesUsed := uint64(logMetrics.InodesUsed.Value()) + result.InodesUsed = &inodesUsed + return result +} diff --git a/pkg/kubelet/stats/log_metrics_provider.go b/pkg/kubelet/stats/log_metrics_provider.go new file mode 100644 index 00000000000..1d31c8b5997 --- /dev/null +++ b/pkg/kubelet/stats/log_metrics_provider.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "k8s.io/kubernetes/pkg/volume" +) + +type LogMetricsService interface { + createLogMetricsProvider(path string) volume.MetricsProvider +} + +type logMetrics struct{} + +func NewLogMetricsService() LogMetricsService { + return logMetrics{} +} + +func (l logMetrics) createLogMetricsProvider(path string) volume.MetricsProvider { + return volume.NewMetricsDu(path) +} diff --git a/pkg/kubelet/stats/stats_provider.go b/pkg/kubelet/stats/stats_provider.go index bf2302920f6..1dc0f00803d 100644 --- a/pkg/kubelet/stats/stats_provider.go +++ b/pkg/kubelet/stats/stats_provider.go @@ -39,8 +39,9 @@ func NewCRIStatsProvider( runtimeCache kubecontainer.RuntimeCache, runtimeService internalapi.RuntimeService, imageService internalapi.ImageManagerService, + logMetricsService LogMetricsService, ) *StatsProvider { - return newStatsProvider(cadvisor, podManager, runtimeCache, newCRIStatsProvider(cadvisor, resourceAnalyzer, runtimeService, imageService)) + return newStatsProvider(cadvisor, podManager, runtimeCache, newCRIStatsProvider(cadvisor, resourceAnalyzer, runtimeService, imageService, logMetricsService)) } // NewCadvisorStatsProvider returns a containerStatsProvider that provides both From ad6bf35c18cf13eb59b8c109c8de816ee4cb2158 Mon Sep 17 00:00:00 2001 From: abhi Date: Tue, 20 Feb 2018 19:50:57 -0800 Subject: [PATCH 2/2] Test cases to verify container log stats The commit contains test case modifications to test and verify changes for container log stats feature. Signed-off-by: abhi --- .../kuberuntime/kuberuntime_container.go | 3 +- .../kuberuntime/kuberuntime_container_test.go | 2 +- pkg/kubelet/stats/cri_stats_provider_test.go | 111 ++++++++++++------ .../stats/log_metrics_provider_test.go | 45 +++++++ 4 files changed, 125 insertions(+), 36 deletions(-) create mode 100644 pkg/kubelet/stats/log_metrics_provider_test.go diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 424a4e3a70a..ad6f0d0686e 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -845,7 +845,8 @@ func (m *kubeGenericRuntimeManager) removeContainerLog(containerID string) error return fmt.Errorf("failed to get container status %q: %v", containerID, err) } labeledInfo := getContainerInfoFromLabels(status.Labels) - path := BuildContainerLogsDirectory(labeledInfo.PodUID, labeledInfo.ContainerName) + annotatedInfo := getContainerInfoFromAnnotations(status.Annotations) + path := buildFullContainerLogsPath(labeledInfo.PodUID, labeledInfo.ContainerName, annotatedInfo.RestartCount) if err := m.osInterface.Remove(path); err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to remove container %q log %q: %v", containerID, path, err) } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go index e99abf75b19..b0ab9a70ccc 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go @@ -61,7 +61,7 @@ func TestRemoveContainer(t *testing.T) { err = m.removeContainer(containerId) assert.NoError(t, err) // Verify container log is removed - expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "12345678", "foo_0.log") + expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "12345678", "foo", "0.log") expectedContainerLogSymlink := legacyLogSymlink(containerId, "foo", "bar", "new") assert.Equal(t, fakeOS.Removes, []string{expectedContainerLogPath, expectedContainerLogSymlink}) // Verify container is removed diff --git a/pkg/kubelet/stats/cri_stats_provider_test.go b/pkg/kubelet/stats/cri_stats_provider_test.go index de4e917df9e..1a790f55631 100644 --- a/pkg/kubelet/stats/cri_stats_provider_test.go +++ b/pkg/kubelet/stats/cri_stats_provider_test.go @@ -24,15 +24,24 @@ import ( cadvisorfs "github.com/google/cadvisor/fs" cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing" + "k8s.io/kubernetes/pkg/kubelet/kuberuntime" "k8s.io/kubernetes/pkg/kubelet/leaky" kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing" serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats" + "k8s.io/kubernetes/pkg/volume" +) + +const ( + offsetInodeUsage = iota + offsetUsage ) func TestCRIListPodStats(t *testing.T) { @@ -68,21 +77,25 @@ func TestCRIListPodStats(t *testing.T) { imageFsInfo = getTestFsInfo(2000) rootFsInfo = getTestFsInfo(1000) - sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns") - container0 = makeFakeContainer(sandbox0, cName0, 0, false) - containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint) - container1 = makeFakeContainer(sandbox0, cName1, 0, false) - containerStats1 = makeFakeContainerStats(container1, unknownMountpoint) + sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns") + container0 = makeFakeContainer(sandbox0, cName0, 0, false) + containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint) + containerLogStats0 = makeFakeLogStats(1000) + container1 = makeFakeContainer(sandbox0, cName1, 0, false) + containerStats1 = makeFakeContainerStats(container1, unknownMountpoint) + containerLogStats1 = makeFakeLogStats(2000) - sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns") - container2 = makeFakeContainer(sandbox1, cName2, 0, false) - containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint) + sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns") + container2 = makeFakeContainer(sandbox1, cName2, 0, false) + containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint) + containerLogStats2 = makeFakeLogStats(3000) - sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns") - container3 = makeFakeContainer(sandbox2, cName3, 0, true) - containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint) - container4 = makeFakeContainer(sandbox2, cName3, 1, false) - containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint) + sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns") + container3 = makeFakeContainer(sandbox2, cName3, 0, true) + containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint) + container4 = makeFakeContainer(sandbox2, cName3, 1, false) + containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint) + containerLogStats4 = makeFakeLogStats(4000) ) var ( @@ -135,13 +148,23 @@ func TestCRIListPodStats(t *testing.T) { PersistentVolumes: persistentVolumes, } + fakeLogStats := map[string]*volume.Metrics{ + kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName0): containerLogStats0, + kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName1): containerLogStats1, + kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox1-uid"), cName2): containerLogStats2, + kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox2-uid"), cName3): containerLogStats4, + } + fakeLogStatsProvider := NewFakeLogMetricsService(fakeLogStats) + provider := NewCRIStatsProvider( mockCadvisor, resourceAnalyzer, mockPodManager, mockRuntimeCache, fakeRuntimeService, - fakeImageService) + fakeImageService, + fakeLogStatsProvider, + ) stats, err := provider.ListPodStats() assert := assert.New(t) @@ -157,7 +180,8 @@ func TestCRIListPodStats(t *testing.T) { assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano()) assert.Equal(2, len(p0.Containers)) - checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1}) + checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1}, + []*volume.Metrics{containerLogStats0, containerLogStats1}) containerStatsMap := make(map[string]statsapi.ContainerStats) for _, s := range p0.Containers { @@ -168,32 +192,32 @@ func TestCRIListPodStats(t *testing.T) { assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano()) checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0]) checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo) - checkCRILogsStats(assert, c0, &rootFsInfo) + checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0) c1 := containerStatsMap[cName1] assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano()) checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0]) checkCRIRootfsStats(assert, c1, containerStats1, nil) - checkCRILogsStats(assert, c1, &rootFsInfo) + checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1) checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network) p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}] assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano()) assert.Equal(1, len(p1.Containers)) - checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2}) + checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2}, []*volume.Metrics{containerLogStats2}) c2 := p1.Containers[0] assert.Equal(cName2, c2.Name) assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano()) checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0]) checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo) - checkCRILogsStats(assert, c2, &rootFsInfo) + checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2) checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network) p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}] assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano()) assert.Equal(1, len(p2.Containers)) - checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4}) + checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4}, []*volume.Metrics{containerLogStats4}) c3 := p2.Containers[0] assert.Equal(cName3, c3.Name) @@ -201,7 +225,7 @@ func TestCRIListPodStats(t *testing.T) { checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0]) checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo) - checkCRILogsStats(assert, c3, &rootFsInfo) + checkCRILogsStats(assert, c3, &rootFsInfo, containerLogStats4) checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network) mockCadvisor.AssertExpectations(t) @@ -214,12 +238,13 @@ func TestCRIImagesFsStats(t *testing.T) { imageFsUsage = makeFakeImageFsUsage(imageFsMountpoint) ) var ( - mockCadvisor = new(cadvisortest.Mock) - mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) - mockPodManager = new(kubepodtest.MockManager) - resourceAnalyzer = new(fakeResourceAnalyzer) - fakeRuntimeService = critest.NewFakeRuntimeService() - fakeImageService = critest.NewFakeImageService() + mockCadvisor = new(cadvisortest.Mock) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + mockPodManager = new(kubepodtest.MockManager) + resourceAnalyzer = new(fakeResourceAnalyzer) + fakeRuntimeService = critest.NewFakeRuntimeService() + fakeImageService = critest.NewFakeImageService() + fakeLogStatsProvider = NewFakeLogMetricsService(nil) ) mockCadvisor.On("GetDirFsInfo", imageFsMountpoint).Return(imageFsInfo, nil) @@ -233,7 +258,9 @@ func TestCRIImagesFsStats(t *testing.T) { mockPodManager, mockRuntimeCache, fakeRuntimeService, - fakeImageService) + fakeImageService, + fakeLogStatsProvider, + ) stats, err := provider.ImageFsStats() assert := assert.New(t) @@ -385,17 +412,21 @@ func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerSta assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed) } -func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo) { +func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo, logStats *volume.Metrics) { assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time) assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes) assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes) assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree) assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes) - assert.Nil(actual.Logs.UsedBytes) - assert.Nil(actual.Logs.InodesUsed) + assert.Equal(uint64(logStats.Used.Value()), *actual.Logs.UsedBytes) + assert.Equal(uint64(logStats.InodesUsed.Value()), *actual.Logs.InodesUsed) } -func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodStats, volumes []statsapi.VolumeStats, containers []*runtimeapi.ContainerStats) { +func checkEphemeralStorageStats(assert *assert.Assertions, + actual statsapi.PodStats, + volumes []statsapi.VolumeStats, + containers []*runtimeapi.ContainerStats, + containerLogStats []*volume.Metrics) { var totalUsed, inodesUsed uint64 for _, container := range containers { totalUsed = totalUsed + container.WritableLayer.UsedBytes.Value @@ -406,8 +437,13 @@ func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodSt totalUsed = totalUsed + *volume.FsStats.UsedBytes inodesUsed = inodesUsed + *volume.FsStats.InodesUsed } - assert.Equal(int(*actual.EphemeralStorage.UsedBytes), int(totalUsed)) - assert.Equal(int(*actual.EphemeralStorage.InodesUsed), int(inodesUsed)) + + for _, logStats := range containerLogStats { + totalUsed = totalUsed + uint64(logStats.Used.Value()) + } + + assert.Equal(int(totalUsed), int(*actual.EphemeralStorage.UsedBytes)) + assert.Equal(int(inodesUsed), int(*actual.EphemeralStorage.InodesUsed)) } func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkStats, expected *cadvisorapiv2.NetworkStats) { @@ -416,3 +452,10 @@ func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkSta assert.Equal(expected.Interfaces[0].TxBytes, *actual.TxBytes) assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors) } + +func makeFakeLogStats(seed int) *volume.Metrics { + m := &volume.Metrics{} + m.Used = resource.NewQuantity(int64(seed+offsetUsage), resource.BinarySI) + m.InodesUsed = resource.NewQuantity(int64(seed+offsetInodeUsage), resource.BinarySI) + return m +} diff --git a/pkg/kubelet/stats/log_metrics_provider_test.go b/pkg/kubelet/stats/log_metrics_provider_test.go new file mode 100644 index 00000000000..499d6130df6 --- /dev/null +++ b/pkg/kubelet/stats/log_metrics_provider_test.go @@ -0,0 +1,45 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "k8s.io/kubernetes/pkg/volume" +) + +type fakeLogMetrics struct { + fakeStats map[string]*volume.Metrics +} + +func NewFakeLogMetricsService(stats map[string]*volume.Metrics) LogMetricsService { + return &fakeLogMetrics{fakeStats: stats} +} + +func (l *fakeLogMetrics) createLogMetricsProvider(path string) volume.MetricsProvider { + return NewFakeMetricsDu(path, l.fakeStats[path]) +} + +type fakeMetricsDu struct { + fakeStats *volume.Metrics +} + +func NewFakeMetricsDu(path string, stats *volume.Metrics) volume.MetricsProvider { + return &fakeMetricsDu{fakeStats: stats} +} + +func (f *fakeMetricsDu) GetMetrics() (*volume.Metrics, error) { + return f.fakeStats, nil +}