github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 05:27:21 +00:00
Code Issues Projects Releases Wiki Activity

Rework hostfs metrics

Browse Source 
Ephemeral storage usage should be calculated by the metrics code,
not the eviction code.
This commit is contained in:
Derek Carr 2020-06-12 16:42:36 -04:00 committed by Joel Smith
parent 39a11744ce
commit acb43c7c4a
13 changed files with 319 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/kubelet/eviction/eviction_manager.go
View File

@ -507,20 +507,11 @@ func (m *managerImpl) podEphemeralStorageLimitEviction(podStats statsapi.PodStat
return false
}
// pod stats api summarizes ephemeral storage usage (container, emptyDir, host[etc-hosts, logs])
podEphemeralStorageTotalUsage := &resource.Quantity{}
var fsStatsSet []fsStatsType
if *m.dedicatedImageFs {
fsStatsSet = []fsStatsType{fsStatsLogs, fsStatsLocalVolumeSource}
} else {
fsStatsSet = []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}
if podStats.EphemeralStorage != nil {
podEphemeralStorageTotalUsage = resource.NewQuantity(int64(*podStats.EphemeralStorage.UsedBytes), resource.BinarySI)
}
podEphemeralUsage, err := podLocalEphemeralStorageUsage(podStats, pod, fsStatsSet, m.etcHostsPath(pod.UID))
if err != nil {
klog.Errorf("eviction manager: error getting pod disk usage %v", err)
return false
}
podEphemeralStorageTotalUsage.Add(podEphemeralUsage[v1.ResourceEphemeralStorage])
podEphemeralStorageLimit := podLimits[v1.ResourceEphemeralStorage]
if podEphemeralStorageTotalUsage.Cmp(podEphemeralStorageLimit) > 0 {
// the total usage of pod exceeds the total size limit of containers, evict the pod

10
pkg/kubelet/kubelet.go
View File

@ -628,6 +628,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
klet.runtimeCache = runtimeCache
// common provider to get host file system usage associated with a pod managed by kubelet
hostStatsProvider := stats.NewHostStatsProvider(kubecontainer.RealOS{}, func(podUID types.UID) string {
return getEtcHostsPath(klet.getPodDir(podUID))
})
if kubeDeps.useLegacyCadvisorStats {
klet.StatsProvider = stats.NewCadvisorStatsProvider(
klet.cadvisor,
@ -635,7 +639,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.podManager,
klet.runtimeCache,
klet.containerRuntime,
klet.statusManager)
klet.statusManager,
hostStatsProvider)
} else {
klet.StatsProvider = stats.NewCRIStatsProvider(
klet.cadvisor,
@ -644,8 +649,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
klet.runtimeCache,
kubeDeps.RemoteRuntimeService,
kubeDeps.RemoteImageService,
stats.NewLogMetricsService(),
kubecontainer.RealOS{})
hostStatsProvider)
}
klet.pleg = pleg.NewGenericPLEG(klet.containerRuntime, plegChannelCapacity, plegRelistPeriod, klet.podCache, clock.RealClock{})

6
pkg/kubelet/kubelet_test.go
View File

@ -250,13 +250,17 @@ func newTestKubeletWithImageList(
volumeStatsAggPeriod := time.Second * 10
kubelet.resourceAnalyzer = serverstats.NewResourceAnalyzer(kubelet, volumeStatsAggPeriod)
fakeHostStatsProvider := stats.NewFakeHostStatsProvider()
kubelet.StatsProvider = stats.NewCadvisorStatsProvider(
kubelet.cadvisor,
kubelet.resourceAnalyzer,
kubelet.podManager,
kubelet.runtimeCache,
fakeRuntime,
kubelet.statusManager)
kubelet.statusManager,
fakeHostStatsProvider,
)
fakeImageGCPolicy := images.ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,

24
pkg/kubelet/stats/cadvisor_stats_provider.go
View File

@ -52,6 +52,8 @@ type cadvisorStatsProvider struct {
imageService kubecontainer.ImageService
// statusProvider is used to get pod metadata
statusProvider status.PodStatusProvider
// hostStatsProvider is used to get pod host stat usage.
hostStatsProvider HostStatsProvider
}
// newCadvisorStatsProvider returns a containerStatsProvider that provides
@ -61,12 +63,14 @@ func newCadvisorStatsProvider(
resourceAnalyzer stats.ResourceAnalyzer,
imageService kubecontainer.ImageService,
statusProvider status.PodStatusProvider,
hostStatsProvider HostStatsProvider,
) containerStatsProvider {
return &cadvisorStatsProvider{
cadvisor: cadvisor,
resourceAnalyzer: resourceAnalyzer,
imageService: imageService,
statusProvider: statusProvider,
cadvisor: cadvisor,
resourceAnalyzer: resourceAnalyzer,
imageService: imageService,
statusProvider: statusProvider,
hostStatsProvider: hostStatsProvider,
}
}
@ -137,7 +141,17 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
copy(ephemeralStats, vstats.EphemeralVolumes)
podStats.VolumeStats = append(append([]statsapi.VolumeStats{}, vstats.EphemeralVolumes...), vstats.PersistentVolumes...)
}
podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo, nil, false)
logStats, err := p.hostStatsProvider.getPodLogStats(podStats.PodRef.Namespace, podStats.PodRef.Name, podUID, &rootFsInfo)
if err != nil {
klog.Errorf("Unable to fetch pod log stats: %v", err)
}
etcHostsStats, err := p.hostStatsProvider.getPodEtcHostsStats(podUID, &rootFsInfo)
if err != nil {
klog.Errorf("unable to fetch pod etc hosts stats: %v", err)
}
podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo, logStats, etcHostsStats, false)
// Lookup the pod-level cgroup's CPU and memory stats
podInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos)
if podInfo != nil {

8
pkg/kubelet/stats/cadvisor_stats_provider_test.go
View File

@ -22,7 +22,7 @@ import (
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
@ -232,7 +232,7 @@ func TestCadvisorListPodStats(t *testing.T) {
resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime, mockStatus)
p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime, mockStatus, NewFakeHostStatsProvider())
pods, err := p.ListPodStats()
assert.NoError(t, err)
@ -400,7 +400,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, nil, nil)
p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, nil, nil, NewFakeHostStatsProvider())
pods, err := p.ListPodCPUAndMemoryStats()
assert.NoError(t, err)
@ -486,7 +486,7 @@ func TestCadvisorImagesFsStats(t *testing.T) {
mockCadvisor.On("ImagesFsInfo").Return(imageFsInfo, nil)
mockRuntime.On("ImageStats").Return(imageStats, nil)
provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime, nil)
provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime, nil, NewFakeHostStatsProvider())
stats, err := provider.ImageFsStats()
assert.NoError(err)

91
pkg/kubelet/stats/cri_stats_provider.go
View File

@ -20,7 +20,6 @@ import (
"errors"
"fmt"
"path"
"path/filepath"
"sort"
"strings"
"sync"
@ -35,8 +34,6 @@ import (
"k8s.io/klog/v2"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
"k8s.io/kubernetes/pkg/kubelet/server/stats"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
)
@ -66,10 +63,8 @@ type criStatsProvider struct {
runtimeService internalapi.RuntimeService
// imageService is used to get the stats of the image filesystem.
imageService internalapi.ImageManagerService
// logMetrics provides the metrics for container logs
logMetricsService LogMetricsService
// osInterface is the interface for syscalls.
osInterface kubecontainer.OSInterface
// hostStatsProvider is used to get the status of the host filesystem consumed by pods.
hostStatsProvider HostStatsProvider
// cpuUsageCache caches the cpu usage for containers.
cpuUsageCache map[string]*cpuUsageRecord
@ -83,16 +78,14 @@ func newCRIStatsProvider(
resourceAnalyzer stats.ResourceAnalyzer,
runtimeService internalapi.RuntimeService,
imageService internalapi.ImageManagerService,
logMetricsService LogMetricsService,
osInterface kubecontainer.OSInterface,
hostStatsProvider HostStatsProvider,
) containerStatsProvider {
return &criStatsProvider{
cadvisor: cadvisor,
resourceAnalyzer: resourceAnalyzer,
runtimeService: runtimeService,
imageService: imageService,
logMetricsService: logMetricsService,
osInterface: osInterface,
hostStatsProvider: hostStatsProvider,
cpuUsageCache: make(map[string]*cpuUsageRecord),
}
}
@ -401,19 +394,22 @@ func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo
if !found {
return
}
podLogDir := kuberuntime.BuildPodLogsDirectory(podNs, podName, podUID)
logStats, err := p.getPodLogStats(podLogDir, rootFsInfo)
logStats, err := p.hostStatsProvider.getPodLogStats(podNs, podName, podUID, rootFsInfo)
if err != nil {
klog.Errorf("Unable to fetch pod log stats for path %s: %v ", podLogDir, err)
klog.Errorf("Unable to fetch pod log stats: %v", err)
// If people do in-place upgrade, there might be pods still using
// the old log path. For those pods, no pod log stats is returned.
// We should continue generating other stats in that case.
// calcEphemeralStorage tolerants logStats == nil.
}
etcHostsStats, err := p.hostStatsProvider.getPodEtcHostsStats(podUID, rootFsInfo)
if err != nil {
klog.Errorf("unable to fetch pod etc hosts stats: %v", err)
}
ephemeralStats := make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes))
copy(ephemeralStats, vstats.EphemeralVolumes)
s.VolumeStats = append(append([]statsapi.VolumeStats{}, vstats.EphemeralVolumes...), vstats.PersistentVolumes...)
s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, true)
s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, etcHostsStats, true)
}
func (p *criStatsProvider) addPodNetworkStats(
@ -581,14 +577,10 @@ func (p *criStatsProvider) makeContainerStats(
// NOTE: This doesn't support the old pod log path, `/var/log/pods/UID`. For containers
// using old log path, empty log stats are returned. This is fine, because we don't
// officially support in-place upgrade anyway.
var (
containerLogPath = kuberuntime.BuildContainerLogsDirectory(meta.GetNamespace(),
meta.GetName(), types.UID(meta.GetUid()), container.GetMetadata().GetName())
err error
)
result.Logs, err = p.getPathFsStats(containerLogPath, rootFsInfo)
var err error
result.Logs, err = p.hostStatsProvider.getPodContainerLogStats(meta.GetNamespace(), meta.GetName(), types.UID(meta.GetUid()), container.GetMetadata().GetName(), rootFsInfo)
if err != nil {
klog.Errorf("Unable to fetch container log stats for path %s: %v ", containerLogPath, err)
klog.Errorf("Unable to fetch container log stats: %v ", err)
}
return result
}
@ -813,58 +805,3 @@ func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) map[strin
}
return stats
}
func (p *criStatsProvider) getPathFsStats(path string, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
m := p.logMetricsService.createLogMetricsProvider(path)
logMetrics, err := m.GetMetrics()
if err != nil {
return nil, err
}
result := &statsapi.FsStats{
Time: metav1.NewTime(rootFsInfo.Timestamp),
AvailableBytes: &rootFsInfo.Available,
CapacityBytes: &rootFsInfo.Capacity,
InodesFree: rootFsInfo.InodesFree,
Inodes: rootFsInfo.Inodes,
}
usedbytes := uint64(logMetrics.Used.Value())
result.UsedBytes = &usedbytes
inodesUsed := uint64(logMetrics.InodesUsed.Value())
result.InodesUsed = &inodesUsed
result.Time = maxUpdateTime(&result.Time, &logMetrics.Time)
return result, nil
}
// getPodLogStats gets stats for logs under the pod log directory. Container logs usually exist
// under the container log directory. However, for some container runtimes, e.g. kata, gvisor,
// they may want to keep some pod level logs, in that case they can put those logs directly under
// the pod log directory. And kubelet will take those logs into account as part of pod ephemeral
// storage.
func (p *criStatsProvider) getPodLogStats(path string, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
files, err := p.osInterface.ReadDir(path)
if err != nil {
return nil, err
}
result := &statsapi.FsStats{
Time: metav1.NewTime(rootFsInfo.Timestamp),
AvailableBytes: &rootFsInfo.Available,
CapacityBytes: &rootFsInfo.Capacity,
InodesFree: rootFsInfo.InodesFree,
Inodes: rootFsInfo.Inodes,
}
for _, f := range files {
if f.IsDir() {
continue
}
// Only include *files* under pod log directory.
fpath := filepath.Join(path, f.Name())
fstats, err := p.getPathFsStats(fpath, rootFsInfo)
if err != nil {
return nil, fmt.Errorf("failed to get fsstats for %q: %v", fpath, err)
}
result.UsedBytes = addUsage(result.UsedBytes, fstats.UsedBytes)
result.InodesUsed = addUsage(result.InodesUsed, fstats.InodesUsed)
result.Time = maxUpdateTime(&result.Time, &fstats.Time)
}
return result, nil
}

25
pkg/kubelet/stats/cri_stats_provider_test.go
View File

@ -192,7 +192,7 @@ func TestCRIListPodStats(t *testing.T) {
PersistentVolumes: persistentVolumes,
}
fakeLogStats := map[string]*volume.Metrics{
fakeStats := map[string]*volume.Metrics{
kuberuntime.BuildContainerLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName0): containerLogStats0,
kuberuntime.BuildContainerLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName1): containerLogStats1,
kuberuntime.BuildContainerLogsDirectory("sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid"), cName2): containerLogStats2,
@ -202,7 +202,6 @@ func TestCRIListPodStats(t *testing.T) {
filepath.Join(kuberuntime.BuildPodLogsDirectory("sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")), podLogName0): podLogStats0,
filepath.Join(kuberuntime.BuildPodLogsDirectory("sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")), podLogName1): podLogStats1,
}
fakeLogStatsProvider := NewFakeLogMetricsService(fakeLogStats)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@ -231,8 +230,7 @@ func TestCRIListPodStats(t *testing.T) {
mockRuntimeCache,
fakeRuntimeService,
fakeImageService,
fakeLogStatsProvider,
fakeOS,
NewFakeHostStatsProviderWithData(fakeStats),
)
stats, err := provider.ListPodStats()
@ -422,8 +420,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
mockRuntimeCache,
fakeRuntimeService,
nil,
nil,
&kubecontainertest.FakeOS{},
NewFakeHostStatsProvider(),
)
stats, err := provider.ListPodCPUAndMemoryStats()
@ -531,13 +528,12 @@ func TestCRIImagesFsStats(t *testing.T) {
imageFsUsage = makeFakeImageFsUsage(imageFsMountpoint)
)
var (
mockCadvisor = new(cadvisortest.Mock)
mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
mockPodManager = new(kubepodtest.MockManager)
resourceAnalyzer = new(fakeResourceAnalyzer)
fakeRuntimeService = critest.NewFakeRuntimeService()
fakeImageService = critest.NewFakeImageService()
fakeLogStatsProvider = NewFakeLogMetricsService(nil)
mockCadvisor = new(cadvisortest.Mock)
mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
mockPodManager = new(kubepodtest.MockManager)
resourceAnalyzer = new(fakeResourceAnalyzer)
fakeRuntimeService = critest.NewFakeRuntimeService()
fakeImageService = critest.NewFakeImageService()
)
mockCadvisor.On("GetDirFsInfo", imageFsMountpoint).Return(imageFsInfo, nil)
@ -552,8 +548,7 @@ func TestCRIImagesFsStats(t *testing.T) {
mockRuntimeCache,
fakeRuntimeService,
fakeImageService,
fakeLogStatsProvider,
&kubecontainertest.FakeOS{},
NewFakeHostStatsProvider(),
)
stats, err := provider.ImageFsStats()

7
pkg/kubelet/stats/helper.go
View File

@ -351,7 +351,7 @@ func uint64Ptr(i uint64) *uint64 {
}
func calcEphemeralStorage(containers []statsapi.ContainerStats, volumes []statsapi.VolumeStats, rootFsInfo *cadvisorapiv2.FsInfo,
podLogStats *statsapi.FsStats, isCRIStatsProvider bool) *statsapi.FsStats {
podLogStats *statsapi.FsStats, etcHostsStats *statsapi.FsStats, isCRIStatsProvider bool) *statsapi.FsStats {
result := &statsapi.FsStats{
Time: metav1.NewTime(rootFsInfo.Timestamp),
AvailableBytes: &rootFsInfo.Available,
@ -372,6 +372,11 @@ func calcEphemeralStorage(containers []statsapi.ContainerStats, volumes []statsa
result.InodesUsed = addUsage(result.InodesUsed, podLogStats.InodesUsed)
result.Time = maxUpdateTime(&result.Time, &podLogStats.Time)
}
if etcHostsStats != nil {
result.UsedBytes = addUsage(result.UsedBytes, etcHostsStats.UsedBytes)
result.InodesUsed = addUsage(result.InodesUsed, etcHostsStats.InodesUsed)
result.Time = maxUpdateTime(&result.Time, &etcHostsStats.Time)
}
return result
}

155
pkg/kubelet/stats/host_stats_provider.go Normal file
View File

@ -0,0 +1,155 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package stats
import (
"fmt"
"path/filepath"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
"k8s.io/kubernetes/pkg/volume"
)
// PodEtcHostsFunc is a function to fetch a etc hosts path by pod uid.
type PodEtcHostsPathFunc func(podUID types.UID) string
// metricsProviderByPath maps a path to its metrics provider
type metricsProviderByPath map[string]volume.MetricsProvider
// HostStatsProvider defines an interface for providing host stats associated with pod.
type HostStatsProvider interface {
// getPodLogStats gets stats associated with pod log usage
getPodLogStats(podNamespace, podName string, podUID types.UID, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error)
// getPodContainerLogStats gets stats associated with container log usage
getPodContainerLogStats(podNamespace, podName string, podUID types.UID, containerName string, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error)
// getPodEtcHostsStats gets stats associated with pod etc-hosts usage
getPodEtcHostsStats(podUID types.UID, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error)
}
type hostStatsProvider struct {
// osInterface is the interface for syscalls.
osInterface kubecontainer.OSInterface
// podEtcHostsPathFunc fetches a pod etc hosts path by uid.
podEtcHostsPathFunc PodEtcHostsPathFunc
}
// NewLogMetricsService returns a new LogMetricsService type struct.
func NewHostStatsProvider(osInterface kubecontainer.OSInterface, podEtcHostsPathFunc PodEtcHostsPathFunc) HostStatsProvider {
return hostStatsProvider{
osInterface: osInterface,
podEtcHostsPathFunc: podEtcHostsPathFunc,
}
}
func (h hostStatsProvider) getPodLogStats(podNamespace, podName string, podUID types.UID, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
metricsByPath, err := h.podLogMetrics(podNamespace, podName, podUID)
if err != nil {
return nil, err
}
return metricsByPathToFsStats(metricsByPath, rootFsInfo)
}
// getPodContainerLogStats gets stats for container
func (h hostStatsProvider) getPodContainerLogStats(podNamespace, podName string, podUID types.UID, containerName string, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
metricsByPath, err := h.podContainerLogMetrics(podNamespace, podName, podUID, containerName)
if err != nil {
return nil, err
}
return metricsByPathToFsStats(metricsByPath, rootFsInfo)
}
// getPodEtcHostsStats gets status for pod etc hosts usage
func (h hostStatsProvider) getPodEtcHostsStats(podUID types.UID, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
metrics := h.podEtcHostsMetrics(podUID)
hostMetrics, err := metrics.GetMetrics()
if err != nil {
return nil, fmt.Errorf("failed to get stats %v", err)
}
result := rootFsInfoToFsStats(rootFsInfo)
usedBytes := uint64(hostMetrics.Used.Value())
inodesUsed := uint64(hostMetrics.InodesUsed.Value())
result.UsedBytes = addUsage(result.UsedBytes, &usedBytes)
result.InodesUsed = addUsage(result.InodesUsed, &inodesUsed)
result.Time = maxUpdateTime(&result.Time, &hostMetrics.Time)
return result, nil
}
func (h hostStatsProvider) podLogMetrics(podNamespace, podName string, podUID types.UID) (metricsProviderByPath, error) {
podLogsDirectoryPath := kuberuntime.BuildPodLogsDirectory(podNamespace, podName, podUID)
return h.fileMetricsByDir(podLogsDirectoryPath)
}
func (h hostStatsProvider) podContainerLogMetrics(podNamespace, podName string, podUID types.UID, containerName string) (metricsProviderByPath, error) {
podContainerLogsDirectoryPath := kuberuntime.BuildContainerLogsDirectory(podNamespace, podName, podUID, containerName)
return h.fileMetricsByDir(podContainerLogsDirectoryPath)
}
func (h hostStatsProvider) podEtcHostsMetrics(podUID types.UID) volume.MetricsProvider {
podEtcHostsPath := h.podEtcHostsPathFunc(podUID)
return volume.NewMetricsDu(podEtcHostsPath)
}
// fileMetricsByDir returns metrics by path for each file under specified directory
func (h hostStatsProvider) fileMetricsByDir(dirname string) (metricsProviderByPath, error) {
files, err := h.osInterface.ReadDir(dirname)
if err != nil {
return nil, err
}
results := metricsProviderByPath{}
for _, f := range files {
if f.IsDir() {
continue
}
// Only include *files* under pod log directory.
fpath := filepath.Join(dirname, f.Name())
results[fpath] = volume.NewMetricsDu(fpath)
}
return results, nil
}
// metricsByPathToFsStats converts a metrics provider by path to fs stats
func metricsByPathToFsStats(metricsByPath metricsProviderByPath, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
result := rootFsInfoToFsStats(rootFsInfo)
for fpath, metrics := range metricsByPath {
hostMetrics, err := metrics.GetMetrics()
if err != nil {
return nil, fmt.Errorf("failed to get fsstats for %q: %v", fpath, err)
}
usedBytes := uint64(hostMetrics.Used.Value())
inodesUsed := uint64(hostMetrics.InodesUsed.Value())
result.UsedBytes = addUsage(result.UsedBytes, &usedBytes)
result.InodesUsed = addUsage(result.InodesUsed, &inodesUsed)
result.Time = maxUpdateTime(&result.Time, &hostMetrics.Time)
}
return result, nil
}
// rootFsInfoToFsStats is a utility to convert rootFsInfo into statsapi.FsStats
func rootFsInfoToFsStats(rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.FsStats {
return &statsapi.FsStats{
Time: metav1.NewTime(rootFsInfo.Timestamp),
AvailableBytes: &rootFsInfo.Available,
CapacityBytes: &rootFsInfo.Capacity,
InodesFree: rootFsInfo.InodesFree,
Inodes: rootFsInfo.Inodes,
}
}

92
pkg/kubelet/stats/host_stats_provider_fake.go Normal file
View File

@ -0,0 +1,92 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package stats
import (
"fmt"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
"k8s.io/apimachinery/pkg/types"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
"k8s.io/kubernetes/pkg/volume"
)
type fakeHostStatsProvider struct {
fakeStats map[string]*volume.Metrics
}
func NewFakeHostStatsProvider() HostStatsProvider {
return &fakeHostStatsProvider{}
}
func NewFakeHostStatsProviderWithData(fakeStats map[string]*volume.Metrics) HostStatsProvider {
return &fakeHostStatsProvider{
fakeStats: fakeStats,
}
}
func (f *fakeHostStatsProvider) getPodLogStats(podNamespace, podName string, podUID types.UID, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
path := kuberuntime.BuildPodLogsDirectory(podNamespace, podName, podUID)
if _, found := f.fakeStats[path]; found {
fmt.Printf("P PATH: %s found\n", path)
}
metricsProvider := NewFakeMetricsDu(path, f.fakeStats[path])
return fakeMetricsProviderToStats(metricsProvider, rootFsInfo)
}
func (f *fakeHostStatsProvider) getPodContainerLogStats(podNamespace, podName string, podUID types.UID, containerName string, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
path := kuberuntime.BuildContainerLogsDirectory(podNamespace, podName, podUID, containerName)
if _, found := f.fakeStats[path]; found {
fmt.Printf("C PATH: %s found\n", path)
}
metricsProvider := NewFakeMetricsDu(path, f.fakeStats[path])
return fakeMetricsProviderToStats(metricsProvider, rootFsInfo)
}
func (f *fakeHostStatsProvider) getPodEtcHostsStats(podUID types.UID, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
return nil, fmt.Errorf("not implemented")
}
func fakeMetricsProviderToStats(metricsProvider volume.MetricsProvider, rootFsInfo *cadvisorapiv2.FsInfo) (*statsapi.FsStats, error) {
hostMetrics, err := metricsProvider.GetMetrics()
if err != nil {
return nil, fmt.Errorf("failed to get stats %v", err)
}
result := rootFsInfoToFsStats(rootFsInfo)
usedBytes := uint64(hostMetrics.Used.Value())
inodesUsed := uint64(hostMetrics.InodesUsed.Value())
result.UsedBytes = addUsage(result.UsedBytes, &usedBytes)
result.InodesUsed = addUsage(result.InodesUsed, &inodesUsed)
result.Time = maxUpdateTime(&result.Time, &hostMetrics.Time)
return result, nil
}
type fakeMetricsDu struct {
fakeStats *volume.Metrics
}
func NewFakeMetricsDu(path string, stats *volume.Metrics) volume.MetricsProvider {
return &fakeMetricsDu{fakeStats: stats}
}
func (f *fakeMetricsDu) GetMetrics() (*volume.Metrics, error) {
if f.fakeStats == nil {
return nil, fmt.Errorf("no stats provided")
}
return f.fakeStats, nil
}

37
pkg/kubelet/stats/log_metrics_provider.go
View File

@ -1,37 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package stats
import (
"k8s.io/kubernetes/pkg/volume"
)
// LogMetricsService defines an interface for providing LogMetrics functionality.
type LogMetricsService interface {
createLogMetricsProvider(path string) volume.MetricsProvider
}
type logMetrics struct{}
// NewLogMetricsService returns a new LogMetricsService type struct.
func NewLogMetricsService() LogMetricsService {
return logMetrics{}
}
func (l logMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
return volume.NewMetricsDu(path)
}

50
pkg/kubelet/stats/log_metrics_provider_test.go
View File

@ -1,50 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package stats
import (
"fmt"
"k8s.io/kubernetes/pkg/volume"
)
type fakeLogMetrics struct {
fakeStats map[string]*volume.Metrics
}
func NewFakeLogMetricsService(stats map[string]*volume.Metrics) LogMetricsService {
return &fakeLogMetrics{fakeStats: stats}
}
func (l *fakeLogMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
return NewFakeMetricsDu(path, l.fakeStats[path])
}
type fakeMetricsDu struct {
fakeStats *volume.Metrics
}
func NewFakeMetricsDu(path string, stats *volume.Metrics) volume.MetricsProvider {
return &fakeMetricsDu{fakeStats: stats}
}
func (f *fakeMetricsDu) GetMetrics() (*volume.Metrics, error) {
if f.fakeStats == nil {
return nil, fmt.Errorf("no stats provided")
}
return f.fakeStats, nil
}

8
pkg/kubelet/stats/provider.go
View File

@ -41,11 +41,10 @@ func NewCRIStatsProvider(
runtimeCache kubecontainer.RuntimeCache,
runtimeService internalapi.RuntimeService,
imageService internalapi.ImageManagerService,
logMetricsService LogMetricsService,
osInterface kubecontainer.OSInterface,
hostStatsProvider HostStatsProvider,
) *Provider {
return newStatsProvider(cadvisor, podManager, runtimeCache, newCRIStatsProvider(cadvisor, resourceAnalyzer,
runtimeService, imageService, logMetricsService, osInterface))
runtimeService, imageService, hostStatsProvider))
}
// NewCadvisorStatsProvider returns a containerStatsProvider that provides both
@ -57,8 +56,9 @@ func NewCadvisorStatsProvider(
runtimeCache kubecontainer.RuntimeCache,
imageService kubecontainer.ImageService,
statusProvider status.PodStatusProvider,
hostStatsProvider HostStatsProvider,
) *Provider {
return newStatsProvider(cadvisor, podManager, runtimeCache, newCadvisorStatsProvider(cadvisor, resourceAnalyzer, imageService, statusProvider))
return newStatsProvider(cadvisor, podManager, runtimeCache, newCadvisorStatsProvider(cadvisor, resourceAnalyzer, imageService, statusProvider, hostStatsProvider))
}
// newStatsProvider returns a new Provider that provides node stats from