github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 11:21:47 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #105585 from fengzixu/improvement-volume-health

Browse Source 
add volume kubelet_volume_stats_health_abnormal to kubelet
This commit is contained in:
Kubernetes Prow Robot 2022-03-15 05:58:11 -07:00 committed by fengzixu
parent 7d675381f8
commit 5cb6fab8f6
7 changed files with 93 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pkg/kubelet/metrics/collectors/volume_stats.go
View File

@ -61,6 +61,12 @@ var (
[]string{"namespace", "persistentvolumeclaim"}, nil,
metrics.ALPHA, "",
)
volumeStatsHealthAbnormalDesc = metrics.NewDesc(
metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusAbnormalKey),
"Abnormal volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy, 0 indicates volume is healthy",
[]string{"namespace", "persistentvolumeclaim"}, nil,
metrics.ALPHA, "")
)
type volumeStatsCollector struct {
@ -85,6 +91,7 @@ func (collector *volumeStatsCollector) DescribeWithStability(ch chan<- *metrics.
ch <- volumeStatsInodesDesc
ch <- volumeStatsInodesFreeDesc
ch <- volumeStatsInodesUsedDesc
ch <- volumeStatsHealthAbnormalDesc
}
// CollectWithStability implements the metrics.StableCollector interface.
@ -95,7 +102,6 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me
}
addGauge := func(desc *metrics.Desc, pvcRef *stats.PVCReference, v float64, lv ...string) {
lv = append([]string{pvcRef.Namespace, pvcRef.Name}, lv...)
ch <- metrics.NewLazyConstMetric(desc, metrics.GaugeValue, v, lv...)
}
allPVCs := sets.String{}
@ -127,3 +133,11 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me
}
}
}
func convertBoolToFloat64(boolVal bool) float64 {
if boolVal {
return 1
}
return 0
}

10
pkg/kubelet/metrics/collectors/volume_stats_test.go
View File

@ -47,6 +47,8 @@ func TestVolumeStatsCollector(t *testing.T) {
# TYPE kubelet_volume_stats_inodes_used gauge
# HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume
# TYPE kubelet_volume_stats_used_bytes gauge
# HELP kubelet_volume_stats_health_status_abnormal [ALPHA] Abnormal volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy, 0 indicates volume is healthy
# TYPE kubelet_volume_stats_health_status_abnormal gauge
`
var (
@ -83,6 +85,9 @@ func TestVolumeStatsCollector(t *testing.T) {
Name: "testpvc",
Namespace: "testns",
},
VolumeHealthStats: &statsapi.VolumeHealthStats{
Abnormal: true,
},
},
},
},
@ -106,6 +111,9 @@ func TestVolumeStatsCollector(t *testing.T) {
Name: "testpvc",
Namespace: "testns",
},
VolumeHealthStats: &statsapi.VolumeHealthStats{
Abnormal: true,
},
},
},
},
@ -118,6 +126,7 @@ func TestVolumeStatsCollector(t *testing.T) {
kubelet_volume_stats_inodes_free{namespace="testns",persistentvolumeclaim="testpvc"} 655344
kubelet_volume_stats_inodes_used{namespace="testns",persistentvolumeclaim="testpvc"} 16
kubelet_volume_stats_used_bytes{namespace="testns",persistentvolumeclaim="testpvc"} 4.21789696e+09
kubelet_volume_stats_health_status_abnormal{namespace="testns",persistentvolumeclaim="testpvc"} 1
`
metrics = []string{
@ -127,6 +136,7 @@ func TestVolumeStatsCollector(t *testing.T) {
"kubelet_volume_stats_inodes_free",
"kubelet_volume_stats_inodes_used",
"kubelet_volume_stats_used_bytes",
"kubelet_volume_stats_health_status_abnormal",
}
)

45
pkg/kubelet/metrics/metrics.go
View File

@ -30,28 +30,29 @@ import (
// This const block defines the metric names for the kubelet metrics.
const (
KubeletSubsystem = "kubelet"
NodeNameKey = "node_name"
NodeLabelKey = "node"
PodWorkerDurationKey = "pod_worker_duration_seconds"
PodStartDurationKey = "pod_start_duration_seconds"
CgroupManagerOperationsKey = "cgroup_manager_duration_seconds"
PodWorkerStartDurationKey = "pod_worker_start_duration_seconds"
PLEGRelistDurationKey = "pleg_relist_duration_seconds"
PLEGDiscardEventsKey = "pleg_discard_events"
PLEGRelistIntervalKey = "pleg_relist_interval_seconds"
PLEGLastSeenKey = "pleg_last_seen_seconds"
EvictionsKey = "evictions"
EvictionStatsAgeKey = "eviction_stats_age_seconds"
PreemptionsKey = "preemptions"
VolumeStatsCapacityBytesKey = "volume_stats_capacity_bytes"
VolumeStatsAvailableBytesKey = "volume_stats_available_bytes"
VolumeStatsUsedBytesKey = "volume_stats_used_bytes"
VolumeStatsInodesKey = "volume_stats_inodes"
VolumeStatsInodesFreeKey = "volume_stats_inodes_free"
VolumeStatsInodesUsedKey = "volume_stats_inodes_used"
RunningPodsKey = "running_pods"
RunningContainersKey = "running_containers"
KubeletSubsystem = "kubelet"
NodeNameKey = "node_name"
NodeLabelKey = "node"
PodWorkerDurationKey = "pod_worker_duration_seconds"
PodStartDurationKey = "pod_start_duration_seconds"
CgroupManagerOperationsKey = "cgroup_manager_duration_seconds"
PodWorkerStartDurationKey = "pod_worker_start_duration_seconds"
PLEGRelistDurationKey = "pleg_relist_duration_seconds"
PLEGDiscardEventsKey = "pleg_discard_events"
PLEGRelistIntervalKey = "pleg_relist_interval_seconds"
PLEGLastSeenKey = "pleg_last_seen_seconds"
EvictionsKey = "evictions"
EvictionStatsAgeKey = "eviction_stats_age_seconds"
PreemptionsKey = "preemptions"
VolumeStatsCapacityBytesKey = "volume_stats_capacity_bytes"
VolumeStatsAvailableBytesKey = "volume_stats_available_bytes"
VolumeStatsUsedBytesKey = "volume_stats_used_bytes"
VolumeStatsInodesKey = "volume_stats_inodes"
VolumeStatsInodesFreeKey = "volume_stats_inodes_free"
VolumeStatsInodesUsedKey = "volume_stats_inodes_used"
VolumeStatsHealthStatusAbnormalKey = "volume_stats_health_status_abnormal"
RunningPodsKey = "running_pods"
RunningContainersKey = "running_containers"
// Metrics keys of remote runtime operations
RuntimeOperationsKey = "runtime_operations_total"
RuntimeOperationsDurationKey = "runtime_operations_duration_seconds"

15
pkg/kubelet/server/stats/volume_stat_calculator.go
View File

@ -177,7 +177,10 @@ func (s *volumeStatCalculator) calcAndStoreStats() {
// parsePodVolumeStats converts (internal) volume.Metrics to (external) stats.VolumeStats structures
func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats.PVCReference, metric *volume.Metrics, volSpec v1.Volume) stats.VolumeStats {
var available, capacity, used, inodes, inodesFree, inodesUsed uint64
var (
available, capacity, used, inodes, inodesFree, inodesUsed uint64
)
if metric.Available != nil {
available = uint64(metric.Available.Value())
}
@ -197,10 +200,18 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats
inodesUsed = uint64(metric.InodesUsed.Value())
}
return stats.VolumeStats{
volumeStats := stats.VolumeStats{
Name: podName,
PVCRef: pvcRef,
FsStats: stats.FsStats{Time: metric.Time, AvailableBytes: &available, CapacityBytes: &capacity,
UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed},
}
if metric.Abnormal != nil {
volumeStats.VolumeHealthStats = &stats.VolumeHealthStats{
Abnormal: *metric.Abnormal,
}
}
return volumeStats
}

24
pkg/kubelet/server/stats/volume_stat_calculator_test.go
View File

@ -128,8 +128,9 @@ func TestPVCRef(t *testing.T) {
assert.Len(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), 4)
// Verify 'vol0' doesn't have a PVC reference
assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
Name: vol0,
FsStats: expectedFSStats(),
Name: vol0,
FsStats: expectedFSStats(),
VolumeHealthStats: expectedVolumeHealthStats(),
})
// Verify 'vol1' has a PVC reference
assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
@ -138,16 +139,18 @@ func TestPVCRef(t *testing.T) {
Name: pvcClaimName0,
Namespace: namespace0,
},
FsStats: expectedFSStats(),
FsStats: expectedFSStats(),
VolumeHealthStats: expectedVolumeHealthStats(),
})
// Verify 'vol2' has a PVC reference
// // Verify 'vol2' has a PVC reference
assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
Name: vol2,
PVCRef: &kubestats.PVCReference{
Name: pvcClaimName1,
Namespace: namespace0,
},
FsStats: expectedBlockStats(),
FsStats: expectedBlockStats(),
VolumeHealthStats: expectedVolumeHealthStats(),
})
// Verify 'vol3' has a PVC reference
assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
@ -156,7 +159,8 @@ func TestPVCRef(t *testing.T) {
Name: pName0 + "-" + vol3,
Namespace: namespace0,
},
FsStats: expectedFSStats(),
FsStats: expectedFSStats(),
VolumeHealthStats: expectedVolumeHealthStats(),
})
}
@ -263,6 +267,13 @@ func expectedFSStats() kubestats.FsStats {
}
}
func expectedVolumeHealthStats() *kubestats.VolumeHealthStats {
metric := expectedMetrics()
return &kubestats.VolumeHealthStats{
Abnormal: *metric.Abnormal,
}
}
// Fake block-volume/metrics provider, block-devices have no inodes
var _ volume.BlockVolume = &fakeBlockVolume{}
@ -283,6 +294,7 @@ func expectedBlockMetrics() *volume.Metrics {
Available: resource.NewQuantity(available, resource.BinarySI),
Capacity: resource.NewQuantity(capacity, resource.BinarySI),
Used: resource.NewQuantity(available-capacity, resource.BinarySI),
Abnormal: &volumeCondition.Abnormal,
}
}

11
staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go
View File

@ -271,6 +271,17 @@ type VolumeStats struct {
// Reference to the PVC, if one exists
// +optional
PVCRef *PVCReference `json:"pvcRef,omitempty"`
// VolumeHealthStats contains data about volume health
// +optional
VolumeHealthStats *VolumeHealthStats `json:"volumeHealthStats,omitempty"`
}
// VolumeHealthStats contains data about volume health.
type VolumeHealthStats struct {
// Normal volumes are available for use and operating optimally.
// An abnormal volume does not meet these criteria.
Abnormal bool `json:"abnormal"`
}
// PVCReference contains enough information to describe the referenced PVC.

5
test/e2e_node/summary_test.go
View File

@ -228,8 +228,9 @@ var _ = SIGDescribe("Summary API [NodeConformance]", func() {
}),
"VolumeStats": gstruct.MatchAllElements(summaryObjectID, gstruct.Elements{
"test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{
"Name": gomega.Equal("test-empty-dir"),
"PVCRef": gomega.BeNil(),
"Name": gomega.Equal("test-empty-dir"),
"PVCRef": gomega.BeNil(),
"VolumeHealthStats": gomega.BeNil(),
"FsStats": gstruct.MatchAllFields(gstruct.Fields{
"Time": recent(maxStatsAge),
"AvailableBytes": fsCapacityBounds,