mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 04:33:26 +00:00
Merge pull request #55447 from jingxu97/Nov/podmetric
Automatic merge from submit-queue (batch tested with PRs 55812, 55752, 55447, 55848, 50984). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add Pod-level local ephemeral storage metric in Summary API This PR adds pod-level ephemeral storage metric into Summary API. Pod-level ephemeral storage usage is the sum of all containers and local ephemeral volume including EmptyDir (if not backed up by memory or hugepages), configueMap, and downwardAPI. Address issue #55978 **Release note**: ```release-note Add pod-level local ephemeral storage metric in Summary API. Pod-level ephemeral storage reports the total filesystem usage for the containers and emptyDir volumes in the measured Pod. ```
This commit is contained in:
commit
94a8d81172
@ -95,6 +95,9 @@ type PodStats struct {
|
|||||||
// +patchMergeKey=name
|
// +patchMergeKey=name
|
||||||
// +patchStrategy=merge
|
// +patchStrategy=merge
|
||||||
VolumeStats []VolumeStats `json:"volume,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
|
VolumeStats []VolumeStats `json:"volume,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
|
||||||
|
// EphemeralStorage reports the total filesystem usage for the containers and emptyDir-backed volumes in the measured Pod.
|
||||||
|
// +optional
|
||||||
|
EphemeralStorage *FsStats `json:"ephemeral-storage,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ContainerStats holds container-level unprocessed sample stats.
|
// ContainerStats holds container-level unprocessed sample stats.
|
||||||
|
@ -42,9 +42,11 @@ type volumeStatCalculator struct {
|
|||||||
latest atomic.Value
|
latest atomic.Value
|
||||||
}
|
}
|
||||||
|
|
||||||
// PodVolumeStats encapsulates all VolumeStats for a pod
|
// PodVolumeStats encapsulates the VolumeStats for a pod.
|
||||||
|
// It consists of two lists, for local ephemeral volumes, and for persistent volumes respectively.
|
||||||
type PodVolumeStats struct {
|
type PodVolumeStats struct {
|
||||||
Volumes []stats.VolumeStats
|
EphemeralVolumes []stats.VolumeStats
|
||||||
|
PersistentVolumes []stats.VolumeStats
|
||||||
}
|
}
|
||||||
|
|
||||||
// newVolumeStatCalculator creates a new VolumeStatCalculator
|
// newVolumeStatCalculator creates a new VolumeStatCalculator
|
||||||
@ -101,7 +103,8 @@ func (s *volumeStatCalculator) calcAndStoreStats() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Call GetMetrics on each Volume and copy the result to a new VolumeStats.FsStats
|
// Call GetMetrics on each Volume and copy the result to a new VolumeStats.FsStats
|
||||||
fsStats := make([]stats.VolumeStats, 0, len(volumes))
|
ephemeralStats := []stats.VolumeStats{}
|
||||||
|
persistentStats := []stats.VolumeStats{}
|
||||||
for name, v := range volumes {
|
for name, v := range volumes {
|
||||||
metric, err := v.GetMetrics()
|
metric, err := v.GetMetrics()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -113,31 +116,38 @@ func (s *volumeStatCalculator) calcAndStoreStats() {
|
|||||||
}
|
}
|
||||||
// Lookup the volume spec and add a 'PVCReference' for volumes that reference a PVC
|
// Lookup the volume spec and add a 'PVCReference' for volumes that reference a PVC
|
||||||
volSpec := volumesSpec[name]
|
volSpec := volumesSpec[name]
|
||||||
|
var pvcRef *stats.PVCReference
|
||||||
if pvcSource := volSpec.PersistentVolumeClaim; pvcSource != nil {
|
if pvcSource := volSpec.PersistentVolumeClaim; pvcSource != nil {
|
||||||
pvcRef := stats.PVCReference{
|
pvcRef = &stats.PVCReference{
|
||||||
Name: pvcSource.ClaimName,
|
Name: pvcSource.ClaimName,
|
||||||
Namespace: s.pod.GetNamespace(),
|
Namespace: s.pod.GetNamespace(),
|
||||||
}
|
}
|
||||||
fsStats = append(fsStats, s.parsePodVolumeStats(name, &pvcRef, metric))
|
|
||||||
// Set the PVC's prometheus metrics
|
// Set the PVC's prometheus metrics
|
||||||
s.setPVCMetrics(&pvcRef, metric)
|
s.setPVCMetrics(pvcRef, metric)
|
||||||
} else {
|
|
||||||
fsStats = append(fsStats, s.parsePodVolumeStats(name, nil, metric))
|
|
||||||
}
|
}
|
||||||
|
volumeStats := s.parsePodVolumeStats(name, pvcRef, metric, volSpec)
|
||||||
|
if isVolumeEphemeral(volSpec) {
|
||||||
|
ephemeralStats = append(ephemeralStats, volumeStats)
|
||||||
|
} else {
|
||||||
|
persistentStats = append(persistentStats, volumeStats)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store the new stats
|
// Store the new stats
|
||||||
s.latest.Store(PodVolumeStats{Volumes: fsStats})
|
s.latest.Store(PodVolumeStats{EphemeralVolumes: ephemeralStats,
|
||||||
|
PersistentVolumes: persistentStats})
|
||||||
}
|
}
|
||||||
|
|
||||||
// parsePodVolumeStats converts (internal) volume.Metrics to (external) stats.VolumeStats structures
|
// parsePodVolumeStats converts (internal) volume.Metrics to (external) stats.VolumeStats structures
|
||||||
func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats.PVCReference, metric *volume.Metrics) stats.VolumeStats {
|
func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats.PVCReference, metric *volume.Metrics, volSpec v1.Volume) stats.VolumeStats {
|
||||||
available := uint64(metric.Available.Value())
|
available := uint64(metric.Available.Value())
|
||||||
capacity := uint64(metric.Capacity.Value())
|
capacity := uint64(metric.Capacity.Value())
|
||||||
used := uint64(metric.Used.Value())
|
used := uint64(metric.Used.Value())
|
||||||
inodes := uint64(metric.Inodes.Value())
|
inodes := uint64(metric.Inodes.Value())
|
||||||
inodesFree := uint64(metric.InodesFree.Value())
|
inodesFree := uint64(metric.InodesFree.Value())
|
||||||
inodesUsed := uint64(metric.InodesUsed.Value())
|
inodesUsed := uint64(metric.InodesUsed.Value())
|
||||||
|
|
||||||
return stats.VolumeStats{
|
return stats.VolumeStats{
|
||||||
Name: podName,
|
Name: podName,
|
||||||
PVCRef: pvcRef,
|
PVCRef: pvcRef,
|
||||||
@ -146,6 +156,14 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isVolumeEphemeral(volume v1.Volume) bool {
|
||||||
|
if (volume.EmptyDir != nil && volume.EmptyDir.Medium == v1.StorageMediumDefault) ||
|
||||||
|
volume.ConfigMap != nil || volume.GitRepo != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// setPVCMetrics sets the given PVC's prometheus metrics to match the given volume.Metrics
|
// setPVCMetrics sets the given PVC's prometheus metrics to match the given volume.Metrics
|
||||||
func (s *volumeStatCalculator) setPVCMetrics(pvcRef *stats.PVCReference, metric *volume.Metrics) {
|
func (s *volumeStatCalculator) setPVCMetrics(pvcRef *stats.PVCReference, metric *volume.Metrics) {
|
||||||
metrics.VolumeStatsAvailableBytes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Available.Value()))
|
metrics.VolumeStatsAvailableBytes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Available.Value()))
|
||||||
|
@ -17,10 +17,11 @@ limitations under the License.
|
|||||||
package stats
|
package stats
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
k8sv1 "k8s.io/api/core/v1"
|
k8sv1 "k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
@ -84,14 +85,14 @@ func TestPVCRef(t *testing.T) {
|
|||||||
statsCalculator.calcAndStoreStats()
|
statsCalculator.calcAndStoreStats()
|
||||||
vs, _ := statsCalculator.GetLatest()
|
vs, _ := statsCalculator.GetLatest()
|
||||||
|
|
||||||
assert.Len(t, vs.Volumes, 2)
|
assert.Len(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), 2)
|
||||||
// Verify 'vol0' doesn't have a PVC reference
|
// Verify 'vol0' doesn't have a PVC reference
|
||||||
assert.Contains(t, vs.Volumes, kubestats.VolumeStats{
|
assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
|
||||||
Name: vol0,
|
Name: vol0,
|
||||||
FsStats: expectedFSStats(),
|
FsStats: expectedFSStats(),
|
||||||
})
|
})
|
||||||
// Verify 'vol1' has a PVC reference
|
// Verify 'vol1' has a PVC reference
|
||||||
assert.Contains(t, vs.Volumes, kubestats.VolumeStats{
|
assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
|
||||||
Name: vol1,
|
Name: vol1,
|
||||||
PVCRef: &kubestats.PVCReference{
|
PVCRef: &kubestats.PVCReference{
|
||||||
Name: pvcClaimName,
|
Name: pvcClaimName,
|
||||||
|
@ -134,15 +134,66 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
|||||||
for _, podStats := range podToStats {
|
for _, podStats := range podToStats {
|
||||||
// Lookup the volume stats for each pod.
|
// Lookup the volume stats for each pod.
|
||||||
podUID := types.UID(podStats.PodRef.UID)
|
podUID := types.UID(podStats.PodRef.UID)
|
||||||
|
var ephemeralStats []statsapi.VolumeStats
|
||||||
if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found {
|
if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found {
|
||||||
podStats.VolumeStats = vstats.Volumes
|
ephemeralStats = make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes))
|
||||||
|
copy(ephemeralStats, vstats.EphemeralVolumes)
|
||||||
|
podStats.VolumeStats = append(vstats.EphemeralVolumes, vstats.PersistentVolumes...)
|
||||||
}
|
}
|
||||||
|
podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo)
|
||||||
result = append(result, *podStats)
|
result = append(result, *podStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func calcEphemeralStorage(containers []statsapi.ContainerStats, volumes []statsapi.VolumeStats, rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.FsStats {
|
||||||
|
result := &statsapi.FsStats{
|
||||||
|
Time: metav1.NewTime(rootFsInfo.Timestamp),
|
||||||
|
AvailableBytes: &rootFsInfo.Available,
|
||||||
|
CapacityBytes: &rootFsInfo.Capacity,
|
||||||
|
InodesFree: rootFsInfo.InodesFree,
|
||||||
|
Inodes: rootFsInfo.Inodes,
|
||||||
|
}
|
||||||
|
for _, container := range containers {
|
||||||
|
addContainerUsage(result, &container)
|
||||||
|
}
|
||||||
|
for _, volume := range volumes {
|
||||||
|
result.UsedBytes = addUsage(result.UsedBytes, volume.FsStats.UsedBytes)
|
||||||
|
result.InodesUsed = addUsage(result.InodesUsed, volume.InodesUsed)
|
||||||
|
result.Time = maxUpdateTime(&result.Time, &volume.FsStats.Time)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func addContainerUsage(stat *statsapi.FsStats, container *statsapi.ContainerStats) {
|
||||||
|
if rootFs := container.Rootfs; rootFs != nil {
|
||||||
|
stat.Time = maxUpdateTime(&stat.Time, &rootFs.Time)
|
||||||
|
stat.InodesUsed = addUsage(stat.InodesUsed, rootFs.InodesUsed)
|
||||||
|
stat.UsedBytes = addUsage(stat.UsedBytes, rootFs.UsedBytes)
|
||||||
|
if logs := container.Logs; logs != nil {
|
||||||
|
stat.UsedBytes = addUsage(stat.UsedBytes, logs.UsedBytes)
|
||||||
|
stat.Time = maxUpdateTime(&stat.Time, &logs.Time)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func maxUpdateTime(first, second *metav1.Time) metav1.Time {
|
||||||
|
if first.Before(second) {
|
||||||
|
return *second
|
||||||
|
}
|
||||||
|
return *first
|
||||||
|
}
|
||||||
|
func addUsage(first, second *uint64) *uint64 {
|
||||||
|
if first == nil {
|
||||||
|
return second
|
||||||
|
} else if second == nil {
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
total := *first + *second
|
||||||
|
return &total
|
||||||
|
}
|
||||||
|
|
||||||
// ImageFsStats returns the stats of the filesystem for storing images.
|
// ImageFsStats returns the stats of the filesystem for storing images.
|
||||||
func (p *cadvisorStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
|
func (p *cadvisorStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
|
||||||
imageFsInfo, err := p.cadvisor.ImagesFsInfo()
|
imageFsInfo, err := p.cadvisor.ImagesFsInfo()
|
||||||
|
@ -27,6 +27,7 @@ import (
|
|||||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||||
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
|
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/leaky"
|
"k8s.io/kubernetes/pkg/kubelet/leaky"
|
||||||
|
serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRemoveTerminatedContainerInfo(t *testing.T) {
|
func TestRemoveTerminatedContainerInfo(t *testing.T) {
|
||||||
@ -79,17 +80,21 @@ func TestCadvisorListPodStats(t *testing.T) {
|
|||||||
namespace2 = "test2"
|
namespace2 = "test2"
|
||||||
)
|
)
|
||||||
const (
|
const (
|
||||||
seedRoot = 0
|
seedRoot = 0
|
||||||
seedRuntime = 100
|
seedRuntime = 100
|
||||||
seedKubelet = 200
|
seedKubelet = 200
|
||||||
seedMisc = 300
|
seedMisc = 300
|
||||||
seedPod0Infra = 1000
|
seedPod0Infra = 1000
|
||||||
seedPod0Container0 = 2000
|
seedPod0Container0 = 2000
|
||||||
seedPod0Container1 = 2001
|
seedPod0Container1 = 2001
|
||||||
seedPod1Infra = 3000
|
seedPod1Infra = 3000
|
||||||
seedPod1Container = 4000
|
seedPod1Container = 4000
|
||||||
seedPod2Infra = 5000
|
seedPod2Infra = 5000
|
||||||
seedPod2Container = 6000
|
seedPod2Container = 6000
|
||||||
|
seedEphemeralVolume1 = 10000
|
||||||
|
seedEphemeralVolume2 = 10001
|
||||||
|
seedPersistentVolume1 = 20000
|
||||||
|
seedPersistentVolume2 = 20001
|
||||||
)
|
)
|
||||||
const (
|
const (
|
||||||
pName0 = "pod0"
|
pName0 = "pod0"
|
||||||
@ -181,7 +186,16 @@ func TestCadvisorListPodStats(t *testing.T) {
|
|||||||
mockRuntime.
|
mockRuntime.
|
||||||
On("ImageStats").Return(&kubecontainer.ImageStats{TotalStorageBytes: 123}, nil)
|
On("ImageStats").Return(&kubecontainer.ImageStats{TotalStorageBytes: 123}, nil)
|
||||||
|
|
||||||
resourceAnalyzer := &fakeResourceAnalyzer{}
|
ephemeralVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedEphemeralVolume1, "ephemeralVolume1"),
|
||||||
|
getPodVolumeStats(seedEphemeralVolume2, "ephemeralVolume2")}
|
||||||
|
persistentVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedPersistentVolume1, "persistentVolume1"),
|
||||||
|
getPodVolumeStats(seedPersistentVolume2, "persistentVolume2")}
|
||||||
|
volumeStats := serverstats.PodVolumeStats{
|
||||||
|
EphemeralVolumes: ephemeralVolumes,
|
||||||
|
PersistentVolumes: persistentVolumes,
|
||||||
|
}
|
||||||
|
|
||||||
|
resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
|
||||||
|
|
||||||
p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime)
|
p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime)
|
||||||
pods, err := p.ListPodStats()
|
pods, err := p.ListPodStats()
|
||||||
@ -213,6 +227,7 @@ func TestCadvisorListPodStats(t *testing.T) {
|
|||||||
|
|
||||||
assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
|
assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
|
||||||
checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)
|
checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)
|
||||||
|
checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, []int{seedEphemeralVolume1, seedEphemeralVolume2}, ps.EphemeralStorage)
|
||||||
|
|
||||||
// Validate Pod1 Results
|
// Validate Pod1 Results
|
||||||
ps, found = indexPods[prf1]
|
ps, found = indexPods[prf1]
|
||||||
|
@ -131,14 +131,16 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
|||||||
// container belongs to.
|
// container belongs to.
|
||||||
ps, found := sandboxIDToPodStats[podSandboxID]
|
ps, found := sandboxIDToPodStats[podSandboxID]
|
||||||
if !found {
|
if !found {
|
||||||
ps = p.makePodStats(podSandbox)
|
ps = buildPodStats(podSandbox)
|
||||||
sandboxIDToPodStats[podSandboxID] = ps
|
sandboxIDToPodStats[podSandboxID] = ps
|
||||||
}
|
}
|
||||||
ps.Containers = append(ps.Containers, *p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo))
|
containerStats := p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo)
|
||||||
|
ps.Containers = append(ps.Containers, *containerStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats))
|
result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats))
|
||||||
for _, s := range sandboxIDToPodStats {
|
for _, s := range sandboxIDToPodStats {
|
||||||
|
p.makePodStorageStats(s, &rootFsInfo)
|
||||||
result = append(result, *s)
|
result = append(result, *s)
|
||||||
}
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
@ -199,8 +201,9 @@ func (p *criStatsProvider) getFsInfo(storageID *runtimeapi.StorageIdentifier) *c
|
|||||||
return &fsInfo
|
return &fsInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *criStatsProvider) makePodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats {
|
// buildPodRef returns a PodStats that identifies the Pod managing cinfo
|
||||||
s := &statsapi.PodStats{
|
func buildPodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats {
|
||||||
|
return &statsapi.PodStats{
|
||||||
PodRef: statsapi.PodReference{
|
PodRef: statsapi.PodReference{
|
||||||
Name: podSandbox.Metadata.Name,
|
Name: podSandbox.Metadata.Name,
|
||||||
UID: podSandbox.Metadata.Uid,
|
UID: podSandbox.Metadata.Uid,
|
||||||
@ -210,9 +213,15 @@ func (p *criStatsProvider) makePodStats(podSandbox *runtimeapi.PodSandbox) *stat
|
|||||||
StartTime: metav1.NewTime(time.Unix(0, podSandbox.CreatedAt)),
|
StartTime: metav1.NewTime(time.Unix(0, podSandbox.CreatedAt)),
|
||||||
// Network stats are not supported by CRI.
|
// Network stats are not supported by CRI.
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.PodStats {
|
||||||
podUID := types.UID(s.PodRef.UID)
|
podUID := types.UID(s.PodRef.UID)
|
||||||
if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found {
|
if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found {
|
||||||
s.VolumeStats = vstats.Volumes
|
ephemeralStats := make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes))
|
||||||
|
copy(ephemeralStats, vstats.EphemeralVolumes)
|
||||||
|
s.VolumeStats = append(vstats.EphemeralVolumes, vstats.PersistentVolumes...)
|
||||||
|
s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo)
|
||||||
}
|
}
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
@ -25,12 +25,14 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
|
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
|
critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
|
||||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
|
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
|
||||||
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
||||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
||||||
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
|
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
|
||||||
kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
|
kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
|
||||||
|
serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCRIListPodStats(t *testing.T) {
|
func TestCRIListPodStats(t *testing.T) {
|
||||||
@ -80,6 +82,13 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
containerStats0, containerStats1, containerStats2, containerStats3, containerStats4,
|
containerStats0, containerStats1, containerStats2, containerStats3, containerStats4,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"})
|
||||||
|
persistentVolumes := makeFakeVolumeStats([]string{"persisVolume1, persisVolumes2"})
|
||||||
|
resourceAnalyzer.podVolumeStats = serverstats.PodVolumeStats{
|
||||||
|
EphemeralVolumes: ephemeralVolumes,
|
||||||
|
PersistentVolumes: persistentVolumes,
|
||||||
|
}
|
||||||
|
|
||||||
provider := NewCRIStatsProvider(
|
provider := NewCRIStatsProvider(
|
||||||
mockCadvisor,
|
mockCadvisor,
|
||||||
resourceAnalyzer,
|
resourceAnalyzer,
|
||||||
@ -102,6 +111,8 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
|
assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
|
||||||
assert.Equal(2, len(p0.Containers))
|
assert.Equal(2, len(p0.Containers))
|
||||||
|
|
||||||
|
checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1})
|
||||||
|
|
||||||
containerStatsMap := make(map[string]statsapi.ContainerStats)
|
containerStatsMap := make(map[string]statsapi.ContainerStats)
|
||||||
for _, s := range p0.Containers {
|
for _, s := range p0.Containers {
|
||||||
containerStatsMap[s.Name] = s
|
containerStatsMap[s.Name] = s
|
||||||
@ -121,6 +132,7 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
|
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
|
||||||
assert.Equal(1, len(p1.Containers))
|
assert.Equal(1, len(p1.Containers))
|
||||||
|
|
||||||
|
checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2})
|
||||||
c2 := p1.Containers[0]
|
c2 := p1.Containers[0]
|
||||||
assert.Equal("container2-name", c2.Name)
|
assert.Equal("container2-name", c2.Name)
|
||||||
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
|
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
|
||||||
@ -132,11 +144,14 @@ func TestCRIListPodStats(t *testing.T) {
|
|||||||
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
|
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
|
||||||
assert.Equal(1, len(p2.Containers))
|
assert.Equal(1, len(p2.Containers))
|
||||||
|
|
||||||
|
checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4})
|
||||||
|
|
||||||
c3 := p2.Containers[0]
|
c3 := p2.Containers[0]
|
||||||
assert.Equal("container3-name", c3.Name)
|
assert.Equal("container3-name", c3.Name)
|
||||||
assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
|
assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
|
||||||
checkCRICPUAndMemoryStats(assert, c3, containerStats4)
|
checkCRICPUAndMemoryStats(assert, c3, containerStats4)
|
||||||
checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
|
checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
|
||||||
|
|
||||||
checkCRILogsStats(assert, c3, &rootFsInfo)
|
checkCRILogsStats(assert, c3, &rootFsInfo)
|
||||||
|
|
||||||
mockCadvisor.AssertExpectations(t)
|
mockCadvisor.AssertExpectations(t)
|
||||||
@ -236,8 +251,8 @@ func makeFakeContainerStats(container *critest.FakeContainer, imageFsUUID string
|
|||||||
WritableLayer: &runtimeapi.FilesystemUsage{
|
WritableLayer: &runtimeapi.FilesystemUsage{
|
||||||
Timestamp: time.Now().UnixNano(),
|
Timestamp: time.Now().UnixNano(),
|
||||||
StorageId: &runtimeapi.StorageIdentifier{Uuid: imageFsUUID},
|
StorageId: &runtimeapi.StorageIdentifier{Uuid: imageFsUUID},
|
||||||
UsedBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
UsedBytes: &runtimeapi.UInt64Value{Value: rand.Uint64() / 100},
|
||||||
InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64() / 100},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if container.State == runtimeapi.ContainerState_CONTAINER_EXITED {
|
if container.State == runtimeapi.ContainerState_CONTAINER_EXITED {
|
||||||
@ -265,6 +280,32 @@ func makeFakeImageFsUsage(fsUUID string) *runtimeapi.FilesystemUsage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func makeFakeVolumeStats(volumeNames []string) []statsapi.VolumeStats {
|
||||||
|
volumes := make([]statsapi.VolumeStats, len(volumeNames))
|
||||||
|
availableBytes := rand.Uint64()
|
||||||
|
capacityBytes := rand.Uint64()
|
||||||
|
usedBytes := rand.Uint64() / 100
|
||||||
|
inodes := rand.Uint64()
|
||||||
|
inodesFree := rand.Uint64()
|
||||||
|
inodesUsed := rand.Uint64() / 100
|
||||||
|
for i, name := range volumeNames {
|
||||||
|
fsStats := statsapi.FsStats{
|
||||||
|
Time: metav1.NewTime(time.Now()),
|
||||||
|
AvailableBytes: &availableBytes,
|
||||||
|
CapacityBytes: &capacityBytes,
|
||||||
|
UsedBytes: &usedBytes,
|
||||||
|
Inodes: &inodes,
|
||||||
|
InodesFree: &inodesFree,
|
||||||
|
InodesUsed: &inodesUsed,
|
||||||
|
}
|
||||||
|
volumes[i] = statsapi.VolumeStats{
|
||||||
|
FsStats: fsStats,
|
||||||
|
Name: name,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return volumes
|
||||||
|
}
|
||||||
|
|
||||||
func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats) {
|
func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats) {
|
||||||
assert.Equal(cs.Cpu.Timestamp, actual.CPU.Time.UnixNano())
|
assert.Equal(cs.Cpu.Timestamp, actual.CPU.Time.UnixNano())
|
||||||
assert.Equal(cs.Cpu.UsageCoreNanoSeconds.Value, *actual.CPU.UsageCoreNanoSeconds)
|
assert.Equal(cs.Cpu.UsageCoreNanoSeconds.Value, *actual.CPU.UsageCoreNanoSeconds)
|
||||||
@ -305,3 +346,18 @@ func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats
|
|||||||
assert.Nil(actual.Logs.UsedBytes)
|
assert.Nil(actual.Logs.UsedBytes)
|
||||||
assert.Nil(actual.Logs.InodesUsed)
|
assert.Nil(actual.Logs.InodesUsed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodStats, volumes []statsapi.VolumeStats, containers []*runtimeapi.ContainerStats) {
|
||||||
|
var totalUsed, inodesUsed uint64
|
||||||
|
for _, container := range containers {
|
||||||
|
totalUsed = totalUsed + container.WritableLayer.UsedBytes.Value
|
||||||
|
inodesUsed = inodesUsed + container.WritableLayer.InodesUsed.Value
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, volume := range volumes {
|
||||||
|
totalUsed = totalUsed + *volume.FsStats.UsedBytes
|
||||||
|
inodesUsed = inodesUsed + *volume.FsStats.InodesUsed
|
||||||
|
}
|
||||||
|
assert.Equal(int(*actual.EphemeralStorage.UsedBytes), int(totalUsed))
|
||||||
|
assert.Equal(int(*actual.EphemeralStorage.InodesUsed), int(inodesUsed))
|
||||||
|
}
|
||||||
|
@ -59,6 +59,7 @@ const (
|
|||||||
offsetFsTotalUsageBytes
|
offsetFsTotalUsageBytes
|
||||||
offsetFsBaseUsageBytes
|
offsetFsBaseUsageBytes
|
||||||
offsetFsInodeUsage
|
offsetFsInodeUsage
|
||||||
|
offsetVolume
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -456,6 +457,28 @@ func getTestFsInfo(seed int) cadvisorapiv2.FsInfo {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getPodVolumeStats(seed int, volumeName string) statsapi.VolumeStats {
|
||||||
|
availableBytes := uint64(seed + offsetFsAvailable)
|
||||||
|
capacityBytes := uint64(seed + offsetFsCapacity)
|
||||||
|
usedBytes := uint64(seed + offsetFsUsage)
|
||||||
|
inodes := uint64(seed + offsetFsInodes)
|
||||||
|
inodesFree := uint64(seed + offsetFsInodesFree)
|
||||||
|
inodesUsed := uint64(seed + offsetFsInodeUsage)
|
||||||
|
fsStats := statsapi.FsStats{
|
||||||
|
Time: metav1.NewTime(time.Now()),
|
||||||
|
AvailableBytes: &availableBytes,
|
||||||
|
CapacityBytes: &capacityBytes,
|
||||||
|
UsedBytes: &usedBytes,
|
||||||
|
Inodes: &inodes,
|
||||||
|
InodesFree: &inodesFree,
|
||||||
|
InodesUsed: &inodesUsed,
|
||||||
|
}
|
||||||
|
return statsapi.VolumeStats{
|
||||||
|
FsStats: fsStats,
|
||||||
|
Name: volumeName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func generateCustomMetricSpec() []cadvisorapiv1.MetricSpec {
|
func generateCustomMetricSpec() []cadvisorapiv1.MetricSpec {
|
||||||
f := fuzz.New().NilChance(0).Funcs(
|
f := fuzz.New().NilChance(0).Funcs(
|
||||||
func(e *cadvisorapiv1.MetricSpec, c fuzz.Continue) {
|
func(e *cadvisorapiv1.MetricSpec, c fuzz.Continue) {
|
||||||
@ -542,6 +565,20 @@ func checkFsStats(t *testing.T, label string, seed int, stats *statsapi.FsStats)
|
|||||||
assert.EqualValues(t, seed+offsetFsInodesFree, *stats.InodesFree, label+".InodesFree")
|
assert.EqualValues(t, seed+offsetFsInodesFree, *stats.InodesFree, label+".InodesFree")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func checkEphemeralStats(t *testing.T, label string, containerSeeds []int, volumeSeeds []int, stats *statsapi.FsStats) {
|
||||||
|
var usedBytes, inodeUsage int
|
||||||
|
for _, cseed := range containerSeeds {
|
||||||
|
usedBytes = usedBytes + cseed + offsetFsTotalUsageBytes
|
||||||
|
inodeUsage += cseed + offsetFsInodeUsage
|
||||||
|
}
|
||||||
|
for _, vseed := range volumeSeeds {
|
||||||
|
usedBytes = usedBytes + vseed + offsetFsUsage
|
||||||
|
inodeUsage += vseed + offsetFsInodeUsage
|
||||||
|
}
|
||||||
|
assert.EqualValues(t, usedBytes, int(*stats.UsedBytes), label+".UsedBytes")
|
||||||
|
assert.EqualValues(t, inodeUsage, int(*stats.InodesUsed), label+".InodesUsed")
|
||||||
|
}
|
||||||
|
|
||||||
type fakeResourceAnalyzer struct {
|
type fakeResourceAnalyzer struct {
|
||||||
podVolumeStats serverstats.PodVolumeStats
|
podVolumeStats serverstats.PodVolumeStats
|
||||||
}
|
}
|
||||||
|
@ -190,7 +190,17 @@ var _ = framework.KubeDescribe("Summary API", func() {
|
|||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
|
"EphemeralStorage": ptrMatchAllFields(gstruct.Fields{
|
||||||
|
"Time": recent(maxStatsAge),
|
||||||
|
"AvailableBytes": fsCapacityBounds,
|
||||||
|
"CapacityBytes": fsCapacityBounds,
|
||||||
|
"UsedBytes": bounded(framework.Kb, 21*framework.Mb),
|
||||||
|
"InodesFree": bounded(1E4, 1E8),
|
||||||
|
"Inodes": bounded(1E4, 1E8),
|
||||||
|
"InodesUsed": bounded(0, 1E8),
|
||||||
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
matchExpectations := ptrMatchAllFields(gstruct.Fields{
|
matchExpectations := ptrMatchAllFields(gstruct.Fields{
|
||||||
"Node": gstruct.MatchAllFields(gstruct.Fields{
|
"Node": gstruct.MatchAllFields(gstruct.Fields{
|
||||||
"NodeName": Equal(framework.TestContext.NodeName),
|
"NodeName": Equal(framework.TestContext.NodeName),
|
||||||
|
Loading…
Reference in New Issue
Block a user