From 9fadd3bd9a3b6e531b5e4b02fb0102b2e15c493a Mon Sep 17 00:00:00 2001 From: NickrenREN Date: Tue, 29 Aug 2017 08:59:42 +0800 Subject: [PATCH] Fix pod local ephemeral storage usage --- pkg/kubelet/eviction/eviction_manager.go | 4 +- pkg/kubelet/eviction/helpers.go | 84 ++++++++++++++++++++---- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index f430b82502b..fb40e98a88a 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -520,13 +520,13 @@ func (m *managerImpl) podEphemeralStorageLimitEviction(podStats statsapi.PodStat } else { fsStatsSet = []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource} } - podUsage, err := podDiskUsage(podStats, pod, fsStatsSet) + podEphemeralUsage, err := podLocalEphemeralStorageUsage(podStats, pod, fsStatsSet) if err != nil { glog.Errorf("eviction manager: error getting pod disk usage %v", err) return false } - podEphemeralStorageTotalUsage.Add(podUsage[resourceDisk]) + podEphemeralStorageTotalUsage.Add(podEphemeralUsage[resourceDisk]) if podEphemeralStorageTotalUsage.Cmp(podLimits[v1.ResourceEphemeralStorage]) > 0 { // the total usage of pod exceeds the total size limit of containers, evict the pod return m.evictPod(pod, v1.ResourceEphemeralStorage, fmt.Sprintf("pod ephemeral local storage usage exceeds the total limit of containers %v", podLimits[v1.ResourceEphemeralStorage])) diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 6cef960232a..34481f96ae5 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -394,8 +394,8 @@ func localVolumeNames(pod *v1.Pod) []string { return result } -// podDiskUsage aggregates pod disk usage and inode consumption for the specified stats to measure. -func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsStatsType) (v1.ResourceList, error) { +// containerUsage aggregates container disk usage and inode consumption for the specified stats to measure. +func containerUsage(podStats statsapi.PodStats, statsToMeasure []fsStatsType) v1.ResourceList { disk := resource.Quantity{Format: resource.BinarySI} inodes := resource.Quantity{Format: resource.BinarySI} for _, container := range podStats.Containers { @@ -408,18 +408,46 @@ func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsSt inodes.Add(*inodeUsage(container.Logs)) } } - if hasFsStatsType(statsToMeasure, fsStatsLocalVolumeSource) { - volumeNames := localVolumeNames(pod) - for _, volumeName := range volumeNames { - for _, volumeStats := range podStats.VolumeStats { - if volumeStats.Name == volumeName { - disk.Add(*diskUsage(&volumeStats.FsStats)) - inodes.Add(*inodeUsage(&volumeStats.FsStats)) - break - } + return v1.ResourceList{ + resourceDisk: disk, + resourceInodes: inodes, + } +} + +// podLocalVolumeUsage aggregates pod local volumes disk usage and inode consumption for the specified stats to measure. +func podLocalVolumeUsage(volumeNames []string, podStats statsapi.PodStats) v1.ResourceList { + disk := resource.Quantity{Format: resource.BinarySI} + inodes := resource.Quantity{Format: resource.BinarySI} + for _, volumeName := range volumeNames { + for _, volumeStats := range podStats.VolumeStats { + if volumeStats.Name == volumeName { + disk.Add(*diskUsage(&volumeStats.FsStats)) + inodes.Add(*inodeUsage(&volumeStats.FsStats)) + break } } } + return v1.ResourceList{ + resourceDisk: disk, + resourceInodes: inodes, + } +} + +// podDiskUsage aggregates pod disk usage and inode consumption for the specified stats to measure. +func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsStatsType) (v1.ResourceList, error) { + disk := resource.Quantity{Format: resource.BinarySI} + inodes := resource.Quantity{Format: resource.BinarySI} + + containerUsageList := containerUsage(podStats, statsToMeasure) + disk.Add(containerUsageList[resourceDisk]) + inodes.Add(containerUsageList[resourceInodes]) + + if hasFsStatsType(statsToMeasure, fsStatsLocalVolumeSource) { + volumeNames := localVolumeNames(pod) + podLocalVolumeUsageList := podLocalVolumeUsage(volumeNames, podStats) + disk.Add(podLocalVolumeUsageList[resourceDisk]) + inodes.Add(podLocalVolumeUsageList[resourceInodes]) + } return v1.ResourceList{ resourceDisk: disk, resourceInodes: inodes, @@ -444,6 +472,40 @@ func podMemoryUsage(podStats statsapi.PodStats) (v1.ResourceList, error) { }, nil } +// localEphemeralVolumeNames returns the set of ephemeral volumes for the pod that are local +func localEphemeralVolumeNames(pod *v1.Pod) []string { + result := []string{} + for _, volume := range pod.Spec.Volumes { + if volume.GitRepo != nil || + (volume.EmptyDir != nil && volume.EmptyDir.Medium != v1.StorageMediumMemory) || + volume.ConfigMap != nil || volume.DownwardAPI != nil { + result = append(result, volume.Name) + } + } + return result +} + +// podLocalEphemeralStorageUsage aggregates pod local ephemeral storage usage and inode consumption for the specified stats to measure. +func podLocalEphemeralStorageUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsStatsType) (v1.ResourceList, error) { + disk := resource.Quantity{Format: resource.BinarySI} + inodes := resource.Quantity{Format: resource.BinarySI} + + containerUsageList := containerUsage(podStats, statsToMeasure) + disk.Add(containerUsageList[resourceDisk]) + inodes.Add(containerUsageList[resourceInodes]) + + if hasFsStatsType(statsToMeasure, fsStatsLocalVolumeSource) { + volumeNames := localEphemeralVolumeNames(pod) + podLocalVolumeUsageList := podLocalVolumeUsage(volumeNames, podStats) + disk.Add(podLocalVolumeUsageList[resourceDisk]) + inodes.Add(podLocalVolumeUsageList[resourceInodes]) + } + return v1.ResourceList{ + resourceDisk: disk, + resourceInodes: inodes, + }, nil +} + // formatThreshold formats a threshold for logging. func formatThreshold(threshold evictionapi.Threshold) string { return fmt.Sprintf("threshold(signal=%v, operator=%v, value=%v, gracePeriod=%v)", threshold.Signal, threshold.Operator, evictionapi.ThresholdValue(threshold.Value), threshold.GracePeriod)