mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 04:33:26 +00:00
Merge pull request #59906 from abhi/log_stats
Automatic merge from submit-queue (batch tested with PRs 54191, 59374, 59824, 55032, 59906). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Adding per container stats for CRI runtimes **What this PR does / why we need it** This commit aims to collect per container log stats. The change was proposed as a part of #55905. The change includes change the log path from /var/pod/<pod uid>/containername_attempt.log to /var/pod/<pod uid>/containername/containername_attempt.log. The logs are collected by reusing volume package to collect metrics from the log path. Fixes #55905 **Special notes for your reviewer:** cc @Random-Liu **Release note:** ``` Adding container log stats for CRI runtimes. ```
This commit is contained in:
commit
742c9b158d
@ -695,7 +695,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
|
|||||||
klet.podManager,
|
klet.podManager,
|
||||||
klet.runtimeCache,
|
klet.runtimeCache,
|
||||||
runtimeService,
|
runtimeService,
|
||||||
imageService)
|
imageService,
|
||||||
|
stats.NewLogMetricsService())
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// rkt uses the legacy, non-CRI, integration. Configure it the old way.
|
// rkt uses the legacy, non-CRI, integration. Configure it the old way.
|
||||||
|
@ -207,7 +207,7 @@ func getStableKey(pod *v1.Pod, container *v1.Container) string {
|
|||||||
|
|
||||||
// buildContainerLogsPath builds log path for container relative to pod logs directory.
|
// buildContainerLogsPath builds log path for container relative to pod logs directory.
|
||||||
func buildContainerLogsPath(containerName string, restartCount int) string {
|
func buildContainerLogsPath(containerName string, restartCount int) string {
|
||||||
return fmt.Sprintf("%s_%d.log", containerName, restartCount)
|
return filepath.Join(containerName, fmt.Sprintf("%d.log", restartCount))
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildFullContainerLogsPath builds absolute log path for container.
|
// buildFullContainerLogsPath builds absolute log path for container.
|
||||||
@ -215,6 +215,11 @@ func buildFullContainerLogsPath(podUID types.UID, containerName string, restartC
|
|||||||
return filepath.Join(buildPodLogsDirectory(podUID), buildContainerLogsPath(containerName, restartCount))
|
return filepath.Join(buildPodLogsDirectory(podUID), buildContainerLogsPath(containerName, restartCount))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BuildContainerLogsDirectory builds absolute log directory path for a container in pod.
|
||||||
|
func BuildContainerLogsDirectory(podUID types.UID, containerName string) string {
|
||||||
|
return filepath.Join(buildPodLogsDirectory(podUID), containerName)
|
||||||
|
}
|
||||||
|
|
||||||
// buildPodLogsDirectory builds absolute log directory path for a pod sandbox.
|
// buildPodLogsDirectory builds absolute log directory path for a pod sandbox.
|
||||||
func buildPodLogsDirectory(podUID types.UID) string {
|
func buildPodLogsDirectory(podUID types.UID) string {
|
||||||
return filepath.Join(podLogsRootDirectory, string(podUID))
|
return filepath.Join(podLogsRootDirectory, string(podUID))
|
||||||
|
@ -190,6 +190,11 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai
|
|||||||
}
|
}
|
||||||
|
|
||||||
command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
|
command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
|
||||||
|
logDir := BuildContainerLogsDirectory(kubetypes.UID(pod.UID), container.Name)
|
||||||
|
err = m.osInterface.MkdirAll(logDir, 0755)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err)
|
||||||
|
}
|
||||||
containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
|
containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
|
||||||
restartCountUint32 := uint32(restartCount)
|
restartCountUint32 := uint32(restartCount)
|
||||||
config := &runtimeapi.ContainerConfig{
|
config := &runtimeapi.ContainerConfig{
|
||||||
|
@ -61,7 +61,7 @@ func TestRemoveContainer(t *testing.T) {
|
|||||||
err = m.removeContainer(containerId)
|
err = m.removeContainer(containerId)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
// Verify container log is removed
|
// Verify container log is removed
|
||||||
expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "12345678", "foo_0.log")
|
expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "12345678", "foo", "0.log")
|
||||||
expectedContainerLogSymlink := legacyLogSymlink(containerId, "foo", "bar", "new")
|
expectedContainerLogSymlink := legacyLogSymlink(containerId, "foo", "bar", "new")
|
||||||
assert.Equal(t, fakeOS.Removes, []string{expectedContainerLogPath, expectedContainerLogSymlink})
|
assert.Equal(t, fakeOS.Removes, []string{expectedContainerLogPath, expectedContainerLogSymlink})
|
||||||
// Verify container is removed
|
// Verify container is removed
|
||||||
|
@ -6,6 +6,7 @@ go_library(
|
|||||||
"cadvisor_stats_provider.go",
|
"cadvisor_stats_provider.go",
|
||||||
"cri_stats_provider.go",
|
"cri_stats_provider.go",
|
||||||
"helper.go",
|
"helper.go",
|
||||||
|
"log_metrics_provider.go",
|
||||||
"stats_provider.go",
|
"stats_provider.go",
|
||||||
] + select({
|
] + select({
|
||||||
"@io_bazel_rules_go//go/platform:android": [
|
"@io_bazel_rules_go//go/platform:android": [
|
||||||
@ -52,11 +53,13 @@ go_library(
|
|||||||
"//pkg/kubelet/cadvisor:go_default_library",
|
"//pkg/kubelet/cadvisor:go_default_library",
|
||||||
"//pkg/kubelet/cm:go_default_library",
|
"//pkg/kubelet/cm:go_default_library",
|
||||||
"//pkg/kubelet/container:go_default_library",
|
"//pkg/kubelet/container:go_default_library",
|
||||||
|
"//pkg/kubelet/kuberuntime:go_default_library",
|
||||||
"//pkg/kubelet/leaky:go_default_library",
|
"//pkg/kubelet/leaky:go_default_library",
|
||||||
"//pkg/kubelet/network:go_default_library",
|
"//pkg/kubelet/network:go_default_library",
|
||||||
"//pkg/kubelet/pod:go_default_library",
|
"//pkg/kubelet/pod:go_default_library",
|
||||||
"//pkg/kubelet/server/stats:go_default_library",
|
"//pkg/kubelet/server/stats:go_default_library",
|
||||||
"//pkg/kubelet/types:go_default_library",
|
"//pkg/kubelet/types:go_default_library",
|
||||||
|
"//pkg/volume:go_default_library",
|
||||||
"//vendor/github.com/golang/glog:go_default_library",
|
"//vendor/github.com/golang/glog:go_default_library",
|
||||||
"//vendor/github.com/golang/protobuf/proto:go_default_library",
|
"//vendor/github.com/golang/protobuf/proto:go_default_library",
|
||||||
"//vendor/github.com/google/cadvisor/fs:go_default_library",
|
"//vendor/github.com/google/cadvisor/fs:go_default_library",
|
||||||
@ -87,6 +90,7 @@ go_test(
|
|||||||
"cadvisor_stats_provider_test.go",
|
"cadvisor_stats_provider_test.go",
|
||||||
"cri_stats_provider_test.go",
|
"cri_stats_provider_test.go",
|
||||||
"helper_test.go",
|
"helper_test.go",
|
||||||
|
"log_metrics_provider_test.go",
|
||||||
"stats_provider_test.go",
|
"stats_provider_test.go",
|
||||||
],
|
],
|
||||||
embed = [":go_default_library"],
|
embed = [":go_default_library"],
|
||||||
@ -97,16 +101,19 @@ go_test(
|
|||||||
"//pkg/kubelet/cadvisor/testing:go_default_library",
|
"//pkg/kubelet/cadvisor/testing:go_default_library",
|
||||||
"//pkg/kubelet/container:go_default_library",
|
"//pkg/kubelet/container:go_default_library",
|
||||||
"//pkg/kubelet/container/testing:go_default_library",
|
"//pkg/kubelet/container/testing:go_default_library",
|
||||||
|
"//pkg/kubelet/kuberuntime:go_default_library",
|
||||||
"//pkg/kubelet/leaky:go_default_library",
|
"//pkg/kubelet/leaky:go_default_library",
|
||||||
"//pkg/kubelet/pod/testing:go_default_library",
|
"//pkg/kubelet/pod/testing:go_default_library",
|
||||||
"//pkg/kubelet/server/stats:go_default_library",
|
"//pkg/kubelet/server/stats:go_default_library",
|
||||||
"//pkg/kubelet/types:go_default_library",
|
"//pkg/kubelet/types:go_default_library",
|
||||||
|
"//pkg/volume:go_default_library",
|
||||||
"//vendor/github.com/google/cadvisor/fs:go_default_library",
|
"//vendor/github.com/google/cadvisor/fs:go_default_library",
|
||||||
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
||||||
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
|
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
|
||||||
"//vendor/github.com/google/gofuzz:go_default_library",
|
"//vendor/github.com/google/gofuzz:go_default_library",
|
||||||
"//vendor/github.com/stretchr/testify/assert:go_default_library",
|
"//vendor/github.com/stretchr/testify/assert:go_default_library",
|
||||||
"//vendor/github.com/stretchr/testify/require:go_default_library",
|
"//vendor/github.com/stretchr/testify/require:go_default_library",
|
||||||
|
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||||
],
|
],
|
||||||
|
@ -35,6 +35,7 @@ import (
|
|||||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||||
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/server/stats"
|
"k8s.io/kubernetes/pkg/kubelet/server/stats"
|
||||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||||
)
|
)
|
||||||
@ -53,6 +54,8 @@ type criStatsProvider struct {
|
|||||||
runtimeService internalapi.RuntimeService
|
runtimeService internalapi.RuntimeService
|
||||||
// imageService is used to get the stats of the image filesystem.
|
// imageService is used to get the stats of the image filesystem.
|
||||||
imageService internalapi.ImageManagerService
|
imageService internalapi.ImageManagerService
|
||||||
|
// logMetrics provides the metrics for container logs
|
||||||
|
logMetricsService LogMetricsService
|
||||||
}
|
}
|
||||||
|
|
||||||
// newCRIStatsProvider returns a containerStatsProvider implementation that
|
// newCRIStatsProvider returns a containerStatsProvider implementation that
|
||||||
@ -62,12 +65,14 @@ func newCRIStatsProvider(
|
|||||||
resourceAnalyzer stats.ResourceAnalyzer,
|
resourceAnalyzer stats.ResourceAnalyzer,
|
||||||
runtimeService internalapi.RuntimeService,
|
runtimeService internalapi.RuntimeService,
|
||||||
imageService internalapi.ImageManagerService,
|
imageService internalapi.ImageManagerService,
|
||||||
|
logMetricsService LogMetricsService,
|
||||||
) containerStatsProvider {
|
) containerStatsProvider {
|
||||||
return &criStatsProvider{
|
return &criStatsProvider{
|
||||||
cadvisor: cadvisor,
|
cadvisor: cadvisor,
|
||||||
resourceAnalyzer: resourceAnalyzer,
|
resourceAnalyzer: resourceAnalyzer,
|
||||||
runtimeService: runtimeService,
|
runtimeService: runtimeService,
|
||||||
imageService: imageService,
|
imageService: imageService,
|
||||||
|
logMetricsService: logMetricsService,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -94,7 +99,6 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
|||||||
for _, s := range podSandboxes {
|
for _, s := range podSandboxes {
|
||||||
podSandboxMap[s.Id] = s
|
podSandboxMap[s.Id] = s
|
||||||
}
|
}
|
||||||
|
|
||||||
// fsIDtoInfo is a map from filesystem id to its stats. This will be used
|
// fsIDtoInfo is a map from filesystem id to its stats. This will be used
|
||||||
// as a cache to avoid querying cAdvisor for the filesystem stats with the
|
// as a cache to avoid querying cAdvisor for the filesystem stats with the
|
||||||
// same filesystem id many times.
|
// same filesystem id many times.
|
||||||
@ -149,7 +153,7 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
|||||||
}
|
}
|
||||||
sandboxIDToPodStats[podSandboxID] = ps
|
sandboxIDToPodStats[podSandboxID] = ps
|
||||||
}
|
}
|
||||||
cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo)
|
cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid())
|
||||||
// If cadvisor stats is available for the container, use it to populate
|
// If cadvisor stats is available for the container, use it to populate
|
||||||
// container stats
|
// container stats
|
||||||
caStats, caFound := caInfos[containerID]
|
caStats, caFound := caInfos[containerID]
|
||||||
@ -277,6 +281,7 @@ func (p *criStatsProvider) makeContainerStats(
|
|||||||
container *runtimeapi.Container,
|
container *runtimeapi.Container,
|
||||||
rootFsInfo *cadvisorapiv2.FsInfo,
|
rootFsInfo *cadvisorapiv2.FsInfo,
|
||||||
fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo,
|
fsIDtoInfo map[runtimeapi.FilesystemIdentifier]*cadvisorapiv2.FsInfo,
|
||||||
|
uid string,
|
||||||
) *statsapi.ContainerStats {
|
) *statsapi.ContainerStats {
|
||||||
result := &statsapi.ContainerStats{
|
result := &statsapi.ContainerStats{
|
||||||
Name: stats.Attributes.Metadata.Name,
|
Name: stats.Attributes.Metadata.Name,
|
||||||
@ -291,17 +296,6 @@ func (p *criStatsProvider) makeContainerStats(
|
|||||||
RSSBytes: proto.Uint64(0),
|
RSSBytes: proto.Uint64(0),
|
||||||
},
|
},
|
||||||
Rootfs: &statsapi.FsStats{},
|
Rootfs: &statsapi.FsStats{},
|
||||||
Logs: &statsapi.FsStats{
|
|
||||||
Time: metav1.NewTime(rootFsInfo.Timestamp),
|
|
||||||
AvailableBytes: &rootFsInfo.Available,
|
|
||||||
CapacityBytes: &rootFsInfo.Capacity,
|
|
||||||
InodesFree: rootFsInfo.InodesFree,
|
|
||||||
Inodes: rootFsInfo.Inodes,
|
|
||||||
// UsedBytes and InodesUsed are unavailable from CRI stats.
|
|
||||||
//
|
|
||||||
// TODO(yguo0905): Get this information from kubelet and
|
|
||||||
// populate the two fields here.
|
|
||||||
},
|
|
||||||
// UserDefinedMetrics is not supported by CRI.
|
// UserDefinedMetrics is not supported by CRI.
|
||||||
}
|
}
|
||||||
if stats.Cpu != nil {
|
if stats.Cpu != nil {
|
||||||
@ -343,7 +337,8 @@ func (p *criStatsProvider) makeContainerStats(
|
|||||||
result.Rootfs.Inodes = imageFsInfo.Inodes
|
result.Rootfs.Inodes = imageFsInfo.Inodes
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
containerLogPath := kuberuntime.BuildContainerLogsDirectory(types.UID(uid), container.GetMetadata().GetName())
|
||||||
|
result.Logs = p.getContainerLogStats(containerLogPath, rootFsInfo)
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -423,3 +418,25 @@ func getCRICadvisorStats(ca cadvisor.Interface) (map[string]cadvisorapiv2.Contai
|
|||||||
}
|
}
|
||||||
return stats, nil
|
return stats, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO Cache the metrics in container log manager
|
||||||
|
func (p *criStatsProvider) getContainerLogStats(path string, rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.FsStats {
|
||||||
|
m := p.logMetricsService.createLogMetricsProvider(path)
|
||||||
|
logMetrics, err := m.GetMetrics()
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("Unable to fetch container log stats for path %s: %v ", path, err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
result := &statsapi.FsStats{
|
||||||
|
Time: metav1.NewTime(rootFsInfo.Timestamp),
|
||||||
|
AvailableBytes: &rootFsInfo.Available,
|
||||||
|
CapacityBytes: &rootFsInfo.Capacity,
|
||||||
|
InodesFree: rootFsInfo.InodesFree,
|
||||||
|
Inodes: rootFsInfo.Inodes,
|
||||||
|
}
|
||||||
|
usedbytes := uint64(logMetrics.Used.Value())
|
||||||
|
result.UsedBytes = &usedbytes
|
||||||
|
inodesUsed := uint64(logMetrics.InodesUsed.Value())
|
||||||
|
result.InodesUsed = &inodesUsed
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
@ -24,15 +24,24 @@ import (
|
|||||||
cadvisorfs "github.com/google/cadvisor/fs"
|
cadvisorfs "github.com/google/cadvisor/fs"
|
||||||
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
|
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||||
critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
|
critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
|
||||||
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
||||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
||||||
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
|
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/leaky"
|
"k8s.io/kubernetes/pkg/kubelet/leaky"
|
||||||
kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
|
kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
|
||||||
serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
|
serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
|
||||||
|
"k8s.io/kubernetes/pkg/volume"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
offsetInodeUsage = iota
|
||||||
|
offsetUsage
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCRIListPodStats(t *testing.T) {
|
func TestCRIListPodStats(t *testing.T) {
|
||||||
@ -68,21 +77,25 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
imageFsInfo = getTestFsInfo(2000)
|
imageFsInfo = getTestFsInfo(2000)
|
||||||
rootFsInfo = getTestFsInfo(1000)
|
rootFsInfo = getTestFsInfo(1000)
|
||||||
|
|
||||||
sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
|
sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
|
||||||
container0 = makeFakeContainer(sandbox0, cName0, 0, false)
|
container0 = makeFakeContainer(sandbox0, cName0, 0, false)
|
||||||
containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint)
|
containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint)
|
||||||
container1 = makeFakeContainer(sandbox0, cName1, 0, false)
|
containerLogStats0 = makeFakeLogStats(1000)
|
||||||
containerStats1 = makeFakeContainerStats(container1, unknownMountpoint)
|
container1 = makeFakeContainer(sandbox0, cName1, 0, false)
|
||||||
|
containerStats1 = makeFakeContainerStats(container1, unknownMountpoint)
|
||||||
|
containerLogStats1 = makeFakeLogStats(2000)
|
||||||
|
|
||||||
sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
|
sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
|
||||||
container2 = makeFakeContainer(sandbox1, cName2, 0, false)
|
container2 = makeFakeContainer(sandbox1, cName2, 0, false)
|
||||||
containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint)
|
containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint)
|
||||||
|
containerLogStats2 = makeFakeLogStats(3000)
|
||||||
|
|
||||||
sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
|
sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
|
||||||
container3 = makeFakeContainer(sandbox2, cName3, 0, true)
|
container3 = makeFakeContainer(sandbox2, cName3, 0, true)
|
||||||
containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint)
|
containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint)
|
||||||
container4 = makeFakeContainer(sandbox2, cName3, 1, false)
|
container4 = makeFakeContainer(sandbox2, cName3, 1, false)
|
||||||
containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
|
containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
|
||||||
|
containerLogStats4 = makeFakeLogStats(4000)
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -135,13 +148,23 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
PersistentVolumes: persistentVolumes,
|
PersistentVolumes: persistentVolumes,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fakeLogStats := map[string]*volume.Metrics{
|
||||||
|
kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName0): containerLogStats0,
|
||||||
|
kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName1): containerLogStats1,
|
||||||
|
kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox1-uid"), cName2): containerLogStats2,
|
||||||
|
kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox2-uid"), cName3): containerLogStats4,
|
||||||
|
}
|
||||||
|
fakeLogStatsProvider := NewFakeLogMetricsService(fakeLogStats)
|
||||||
|
|
||||||
provider := NewCRIStatsProvider(
|
provider := NewCRIStatsProvider(
|
||||||
mockCadvisor,
|
mockCadvisor,
|
||||||
resourceAnalyzer,
|
resourceAnalyzer,
|
||||||
mockPodManager,
|
mockPodManager,
|
||||||
mockRuntimeCache,
|
mockRuntimeCache,
|
||||||
fakeRuntimeService,
|
fakeRuntimeService,
|
||||||
fakeImageService)
|
fakeImageService,
|
||||||
|
fakeLogStatsProvider,
|
||||||
|
)
|
||||||
|
|
||||||
stats, err := provider.ListPodStats()
|
stats, err := provider.ListPodStats()
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
@ -157,7 +180,8 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
|
assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
|
||||||
assert.Equal(2, len(p0.Containers))
|
assert.Equal(2, len(p0.Containers))
|
||||||
|
|
||||||
checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1})
|
checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1},
|
||||||
|
[]*volume.Metrics{containerLogStats0, containerLogStats1})
|
||||||
|
|
||||||
containerStatsMap := make(map[string]statsapi.ContainerStats)
|
containerStatsMap := make(map[string]statsapi.ContainerStats)
|
||||||
for _, s := range p0.Containers {
|
for _, s := range p0.Containers {
|
||||||
@ -168,32 +192,32 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
|
assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
|
||||||
checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
|
checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
|
||||||
checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
|
checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
|
||||||
checkCRILogsStats(assert, c0, &rootFsInfo)
|
checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0)
|
||||||
c1 := containerStatsMap[cName1]
|
c1 := containerStatsMap[cName1]
|
||||||
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
|
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
|
||||||
checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
|
checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
|
||||||
checkCRIRootfsStats(assert, c1, containerStats1, nil)
|
checkCRIRootfsStats(assert, c1, containerStats1, nil)
|
||||||
checkCRILogsStats(assert, c1, &rootFsInfo)
|
checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1)
|
||||||
checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
|
checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
|
||||||
|
|
||||||
p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
|
p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
|
||||||
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
|
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
|
||||||
assert.Equal(1, len(p1.Containers))
|
assert.Equal(1, len(p1.Containers))
|
||||||
|
|
||||||
checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2})
|
checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2}, []*volume.Metrics{containerLogStats2})
|
||||||
c2 := p1.Containers[0]
|
c2 := p1.Containers[0]
|
||||||
assert.Equal(cName2, c2.Name)
|
assert.Equal(cName2, c2.Name)
|
||||||
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
|
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
|
||||||
checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
|
checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
|
||||||
checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
|
checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
|
||||||
checkCRILogsStats(assert, c2, &rootFsInfo)
|
checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2)
|
||||||
checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
|
checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
|
||||||
|
|
||||||
p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
|
p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
|
||||||
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
|
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
|
||||||
assert.Equal(1, len(p2.Containers))
|
assert.Equal(1, len(p2.Containers))
|
||||||
|
|
||||||
checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4})
|
checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4}, []*volume.Metrics{containerLogStats4})
|
||||||
|
|
||||||
c3 := p2.Containers[0]
|
c3 := p2.Containers[0]
|
||||||
assert.Equal(cName3, c3.Name)
|
assert.Equal(cName3, c3.Name)
|
||||||
@ -201,7 +225,7 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
|
checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
|
||||||
checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
|
checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
|
||||||
|
|
||||||
checkCRILogsStats(assert, c3, &rootFsInfo)
|
checkCRILogsStats(assert, c3, &rootFsInfo, containerLogStats4)
|
||||||
checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
|
checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
|
||||||
|
|
||||||
mockCadvisor.AssertExpectations(t)
|
mockCadvisor.AssertExpectations(t)
|
||||||
@ -214,12 +238,13 @@ func TestCRIImagesFsStats(t *testing.T) {
|
|||||||
imageFsUsage = makeFakeImageFsUsage(imageFsMountpoint)
|
imageFsUsage = makeFakeImageFsUsage(imageFsMountpoint)
|
||||||
)
|
)
|
||||||
var (
|
var (
|
||||||
mockCadvisor = new(cadvisortest.Mock)
|
mockCadvisor = new(cadvisortest.Mock)
|
||||||
mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
|
mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
|
||||||
mockPodManager = new(kubepodtest.MockManager)
|
mockPodManager = new(kubepodtest.MockManager)
|
||||||
resourceAnalyzer = new(fakeResourceAnalyzer)
|
resourceAnalyzer = new(fakeResourceAnalyzer)
|
||||||
fakeRuntimeService = critest.NewFakeRuntimeService()
|
fakeRuntimeService = critest.NewFakeRuntimeService()
|
||||||
fakeImageService = critest.NewFakeImageService()
|
fakeImageService = critest.NewFakeImageService()
|
||||||
|
fakeLogStatsProvider = NewFakeLogMetricsService(nil)
|
||||||
)
|
)
|
||||||
|
|
||||||
mockCadvisor.On("GetDirFsInfo", imageFsMountpoint).Return(imageFsInfo, nil)
|
mockCadvisor.On("GetDirFsInfo", imageFsMountpoint).Return(imageFsInfo, nil)
|
||||||
@ -233,7 +258,9 @@ func TestCRIImagesFsStats(t *testing.T) {
|
|||||||
mockPodManager,
|
mockPodManager,
|
||||||
mockRuntimeCache,
|
mockRuntimeCache,
|
||||||
fakeRuntimeService,
|
fakeRuntimeService,
|
||||||
fakeImageService)
|
fakeImageService,
|
||||||
|
fakeLogStatsProvider,
|
||||||
|
)
|
||||||
|
|
||||||
stats, err := provider.ImageFsStats()
|
stats, err := provider.ImageFsStats()
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
@ -385,17 +412,21 @@ func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerSta
|
|||||||
assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed)
|
assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed)
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo) {
|
func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo, logStats *volume.Metrics) {
|
||||||
assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time)
|
assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time)
|
||||||
assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes)
|
assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes)
|
||||||
assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes)
|
assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes)
|
||||||
assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree)
|
assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree)
|
||||||
assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes)
|
assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes)
|
||||||
assert.Nil(actual.Logs.UsedBytes)
|
assert.Equal(uint64(logStats.Used.Value()), *actual.Logs.UsedBytes)
|
||||||
assert.Nil(actual.Logs.InodesUsed)
|
assert.Equal(uint64(logStats.InodesUsed.Value()), *actual.Logs.InodesUsed)
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodStats, volumes []statsapi.VolumeStats, containers []*runtimeapi.ContainerStats) {
|
func checkEphemeralStorageStats(assert *assert.Assertions,
|
||||||
|
actual statsapi.PodStats,
|
||||||
|
volumes []statsapi.VolumeStats,
|
||||||
|
containers []*runtimeapi.ContainerStats,
|
||||||
|
containerLogStats []*volume.Metrics) {
|
||||||
var totalUsed, inodesUsed uint64
|
var totalUsed, inodesUsed uint64
|
||||||
for _, container := range containers {
|
for _, container := range containers {
|
||||||
totalUsed = totalUsed + container.WritableLayer.UsedBytes.Value
|
totalUsed = totalUsed + container.WritableLayer.UsedBytes.Value
|
||||||
@ -406,8 +437,13 @@ func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodSt
|
|||||||
totalUsed = totalUsed + *volume.FsStats.UsedBytes
|
totalUsed = totalUsed + *volume.FsStats.UsedBytes
|
||||||
inodesUsed = inodesUsed + *volume.FsStats.InodesUsed
|
inodesUsed = inodesUsed + *volume.FsStats.InodesUsed
|
||||||
}
|
}
|
||||||
assert.Equal(int(*actual.EphemeralStorage.UsedBytes), int(totalUsed))
|
|
||||||
assert.Equal(int(*actual.EphemeralStorage.InodesUsed), int(inodesUsed))
|
for _, logStats := range containerLogStats {
|
||||||
|
totalUsed = totalUsed + uint64(logStats.Used.Value())
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(int(totalUsed), int(*actual.EphemeralStorage.UsedBytes))
|
||||||
|
assert.Equal(int(inodesUsed), int(*actual.EphemeralStorage.InodesUsed))
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkStats, expected *cadvisorapiv2.NetworkStats) {
|
func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkStats, expected *cadvisorapiv2.NetworkStats) {
|
||||||
@ -416,3 +452,10 @@ func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkSta
|
|||||||
assert.Equal(expected.Interfaces[0].TxBytes, *actual.TxBytes)
|
assert.Equal(expected.Interfaces[0].TxBytes, *actual.TxBytes)
|
||||||
assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors)
|
assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func makeFakeLogStats(seed int) *volume.Metrics {
|
||||||
|
m := &volume.Metrics{}
|
||||||
|
m.Used = resource.NewQuantity(int64(seed+offsetUsage), resource.BinarySI)
|
||||||
|
m.InodesUsed = resource.NewQuantity(int64(seed+offsetInodeUsage), resource.BinarySI)
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
35
pkg/kubelet/stats/log_metrics_provider.go
Normal file
35
pkg/kubelet/stats/log_metrics_provider.go
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2018 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package stats
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/kubernetes/pkg/volume"
|
||||||
|
)
|
||||||
|
|
||||||
|
type LogMetricsService interface {
|
||||||
|
createLogMetricsProvider(path string) volume.MetricsProvider
|
||||||
|
}
|
||||||
|
|
||||||
|
type logMetrics struct{}
|
||||||
|
|
||||||
|
func NewLogMetricsService() LogMetricsService {
|
||||||
|
return logMetrics{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l logMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
|
||||||
|
return volume.NewMetricsDu(path)
|
||||||
|
}
|
45
pkg/kubelet/stats/log_metrics_provider_test.go
Normal file
45
pkg/kubelet/stats/log_metrics_provider_test.go
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2018 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package stats
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/kubernetes/pkg/volume"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeLogMetrics struct {
|
||||||
|
fakeStats map[string]*volume.Metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFakeLogMetricsService(stats map[string]*volume.Metrics) LogMetricsService {
|
||||||
|
return &fakeLogMetrics{fakeStats: stats}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *fakeLogMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
|
||||||
|
return NewFakeMetricsDu(path, l.fakeStats[path])
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeMetricsDu struct {
|
||||||
|
fakeStats *volume.Metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFakeMetricsDu(path string, stats *volume.Metrics) volume.MetricsProvider {
|
||||||
|
return &fakeMetricsDu{fakeStats: stats}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeMetricsDu) GetMetrics() (*volume.Metrics, error) {
|
||||||
|
return f.fakeStats, nil
|
||||||
|
}
|
@ -39,8 +39,9 @@ func NewCRIStatsProvider(
|
|||||||
runtimeCache kubecontainer.RuntimeCache,
|
runtimeCache kubecontainer.RuntimeCache,
|
||||||
runtimeService internalapi.RuntimeService,
|
runtimeService internalapi.RuntimeService,
|
||||||
imageService internalapi.ImageManagerService,
|
imageService internalapi.ImageManagerService,
|
||||||
|
logMetricsService LogMetricsService,
|
||||||
) *StatsProvider {
|
) *StatsProvider {
|
||||||
return newStatsProvider(cadvisor, podManager, runtimeCache, newCRIStatsProvider(cadvisor, resourceAnalyzer, runtimeService, imageService))
|
return newStatsProvider(cadvisor, podManager, runtimeCache, newCRIStatsProvider(cadvisor, resourceAnalyzer, runtimeService, imageService, logMetricsService))
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewCadvisorStatsProvider returns a containerStatsProvider that provides both
|
// NewCadvisorStatsProvider returns a containerStatsProvider that provides both
|
||||||
|
Loading…
Reference in New Issue
Block a user