Merge pull request #105585 from fengzixu/improvement-volume-health

add volume kubelet_volume_stats_health_abnormal to kubelet

pkg/kubelet/metrics/collectors/volume_stats.go

@@ -61,6 +61,12 @@ var (
 		[]string{"namespace", "persistentvolumeclaim"}, nil,
 		metrics.ALPHA, "",
 	)
+
+	volumeStatsHealthAbnormalDesc = metrics.NewDesc(
+		metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusAbnormalKey),
+		"Abnormal volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy, 0 indicates volume is healthy",
+		[]string{"namespace", "persistentvolumeclaim"}, nil,
+		metrics.ALPHA, "")
 )
 
 type volumeStatsCollector struct {
@@ -85,6 +91,7 @@ func (collector *volumeStatsCollector) DescribeWithStability(ch chan<- *metrics.
 	ch <- volumeStatsInodesDesc
 	ch <- volumeStatsInodesFreeDesc
 	ch <- volumeStatsInodesUsedDesc
+	ch <- volumeStatsHealthAbnormalDesc
 }
 
 // CollectWithStability implements the metrics.StableCollector interface.
@@ -95,7 +102,6 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me
 	}
 	addGauge := func(desc *metrics.Desc, pvcRef *stats.PVCReference, v float64, lv ...string) {
 		lv = append([]string{pvcRef.Namespace, pvcRef.Name}, lv...)
-
 		ch <- metrics.NewLazyConstMetric(desc, metrics.GaugeValue, v, lv...)
 	}
 	allPVCs := sets.String{}
@@ -127,3 +133,11 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me
 		}
 	}
 }
+
+func convertBoolToFloat64(boolVal bool) float64 {
+	if boolVal {
+		return 1
+	}
+
+	return 0
+}

pkg/kubelet/metrics/collectors/volume_stats_test.go

@@ -47,6 +47,8 @@ func TestVolumeStatsCollector(t *testing.T) {
 		# TYPE kubelet_volume_stats_inodes_used gauge
 		# HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume
 		# TYPE kubelet_volume_stats_used_bytes gauge
+		# HELP kubelet_volume_stats_health_status_abnormal [ALPHA] Abnormal volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy, 0 indicates volume is healthy
+		# TYPE kubelet_volume_stats_health_status_abnormal gauge
 	`
 
 	var (
@@ -83,6 +85,9 @@ func TestVolumeStatsCollector(t *testing.T) {
 							Name:      "testpvc",
 							Namespace: "testns",
 						},
+						VolumeHealthStats: &statsapi.VolumeHealthStats{
+							Abnormal: true,
+						},
 					},
 				},
 			},
@@ -106,6 +111,9 @@ func TestVolumeStatsCollector(t *testing.T) {
 							Name:      "testpvc",
 							Namespace: "testns",
 						},
+						VolumeHealthStats: &statsapi.VolumeHealthStats{
+							Abnormal: true,
+						},
 					},
 				},
 			},
@@ -118,6 +126,7 @@ func TestVolumeStatsCollector(t *testing.T) {
 			kubelet_volume_stats_inodes_free{namespace="testns",persistentvolumeclaim="testpvc"} 655344
 			kubelet_volume_stats_inodes_used{namespace="testns",persistentvolumeclaim="testpvc"} 16
 			kubelet_volume_stats_used_bytes{namespace="testns",persistentvolumeclaim="testpvc"} 4.21789696e+09
+			kubelet_volume_stats_health_status_abnormal{namespace="testns",persistentvolumeclaim="testpvc"} 1
 			`
 
 		metrics = []string{
@@ -127,6 +136,7 @@ func TestVolumeStatsCollector(t *testing.T) {
 			"kubelet_volume_stats_inodes_free",
 			"kubelet_volume_stats_inodes_used",
 			"kubelet_volume_stats_used_bytes",
+			"kubelet_volume_stats_health_status_abnormal",
 		}
 	)
 

pkg/kubelet/metrics/metrics.go

@@ -30,28 +30,29 @@ import (
 
 // This const block defines the metric names for the kubelet metrics.
 const (
-	KubeletSubsystem             = "kubelet"
+	KubeletSubsystem                   = "kubelet"
-	NodeNameKey                  = "node_name"
+	NodeNameKey                        = "node_name"
-	NodeLabelKey                 = "node"
+	NodeLabelKey                       = "node"
-	PodWorkerDurationKey         = "pod_worker_duration_seconds"
+	PodWorkerDurationKey               = "pod_worker_duration_seconds"
-	PodStartDurationKey          = "pod_start_duration_seconds"
+	PodStartDurationKey                = "pod_start_duration_seconds"
-	CgroupManagerOperationsKey   = "cgroup_manager_duration_seconds"
+	CgroupManagerOperationsKey         = "cgroup_manager_duration_seconds"
-	PodWorkerStartDurationKey    = "pod_worker_start_duration_seconds"
+	PodWorkerStartDurationKey          = "pod_worker_start_duration_seconds"
-	PLEGRelistDurationKey        = "pleg_relist_duration_seconds"
+	PLEGRelistDurationKey              = "pleg_relist_duration_seconds"
-	PLEGDiscardEventsKey         = "pleg_discard_events"
+	PLEGDiscardEventsKey               = "pleg_discard_events"
-	PLEGRelistIntervalKey        = "pleg_relist_interval_seconds"
+	PLEGRelistIntervalKey              = "pleg_relist_interval_seconds"
-	PLEGLastSeenKey              = "pleg_last_seen_seconds"
+	PLEGLastSeenKey                    = "pleg_last_seen_seconds"
-	EvictionsKey                 = "evictions"
+	EvictionsKey                       = "evictions"
-	EvictionStatsAgeKey          = "eviction_stats_age_seconds"
+	EvictionStatsAgeKey                = "eviction_stats_age_seconds"
-	PreemptionsKey               = "preemptions"
+	PreemptionsKey                     = "preemptions"
-	VolumeStatsCapacityBytesKey  = "volume_stats_capacity_bytes"
+	VolumeStatsCapacityBytesKey        = "volume_stats_capacity_bytes"
-	VolumeStatsAvailableBytesKey = "volume_stats_available_bytes"
+	VolumeStatsAvailableBytesKey       = "volume_stats_available_bytes"
-	VolumeStatsUsedBytesKey      = "volume_stats_used_bytes"
+	VolumeStatsUsedBytesKey            = "volume_stats_used_bytes"
-	VolumeStatsInodesKey         = "volume_stats_inodes"
+	VolumeStatsInodesKey               = "volume_stats_inodes"
-	VolumeStatsInodesFreeKey     = "volume_stats_inodes_free"
+	VolumeStatsInodesFreeKey           = "volume_stats_inodes_free"
-	VolumeStatsInodesUsedKey     = "volume_stats_inodes_used"
+	VolumeStatsInodesUsedKey           = "volume_stats_inodes_used"
-	RunningPodsKey               = "running_pods"
+	VolumeStatsHealthStatusAbnormalKey = "volume_stats_health_status_abnormal"
-	RunningContainersKey         = "running_containers"
+	RunningPodsKey                     = "running_pods"
+	RunningContainersKey               = "running_containers"
 	// Metrics keys of remote runtime operations
 	RuntimeOperationsKey         = "runtime_operations_total"
 	RuntimeOperationsDurationKey = "runtime_operations_duration_seconds"

pkg/kubelet/server/stats/volume_stat_calculator.go

@@ -177,7 +177,10 @@ func (s *volumeStatCalculator) calcAndStoreStats() {
 // parsePodVolumeStats converts (internal) volume.Metrics to (external) stats.VolumeStats structures
 func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats.PVCReference, metric *volume.Metrics, volSpec v1.Volume) stats.VolumeStats {
 
-	var available, capacity, used, inodes, inodesFree, inodesUsed uint64
+	var (
+		available, capacity, used, inodes, inodesFree, inodesUsed uint64
+	)
+
 	if metric.Available != nil {
 		available = uint64(metric.Available.Value())
 	}
@@ -197,10 +200,18 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats
 		inodesUsed = uint64(metric.InodesUsed.Value())
 	}
 
-	return stats.VolumeStats{
+	volumeStats := stats.VolumeStats{
 		Name:   podName,
 		PVCRef: pvcRef,
 		FsStats: stats.FsStats{Time: metric.Time, AvailableBytes: &available, CapacityBytes: &capacity,
 			UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed},
 	}
+
+	if metric.Abnormal != nil {
+		volumeStats.VolumeHealthStats = &stats.VolumeHealthStats{
+			Abnormal: *metric.Abnormal,
+		}
+	}
+
+	return volumeStats
 }

pkg/kubelet/server/stats/volume_stat_calculator_test.go

@@ -128,8 +128,9 @@ func TestPVCRef(t *testing.T) {
 	assert.Len(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), 4)
 	// Verify 'vol0' doesn't have a PVC reference
 	assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
-		Name:    vol0,
+		Name:              vol0,
-		FsStats: expectedFSStats(),
+		FsStats:           expectedFSStats(),
+		VolumeHealthStats: expectedVolumeHealthStats(),
 	})
 	// Verify 'vol1' has a PVC reference
 	assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
@@ -138,16 +139,18 @@ func TestPVCRef(t *testing.T) {
 			Name:      pvcClaimName0,
 			Namespace: namespace0,
 		},
-		FsStats: expectedFSStats(),
+		FsStats:           expectedFSStats(),
+		VolumeHealthStats: expectedVolumeHealthStats(),
 	})
-	// Verify 'vol2' has a PVC reference
+	// // Verify 'vol2' has a PVC reference
 	assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
 		Name: vol2,
 		PVCRef: &kubestats.PVCReference{
 			Name:      pvcClaimName1,
 			Namespace: namespace0,
 		},
-		FsStats: expectedBlockStats(),
+		FsStats:           expectedBlockStats(),
+		VolumeHealthStats: expectedVolumeHealthStats(),
 	})
 	// Verify 'vol3' has a PVC reference
 	assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{
@@ -156,7 +159,8 @@ func TestPVCRef(t *testing.T) {
 			Name:      pName0 + "-" + vol3,
 			Namespace: namespace0,
 		},
-		FsStats: expectedFSStats(),
+		FsStats:           expectedFSStats(),
+		VolumeHealthStats: expectedVolumeHealthStats(),
 	})
 }
 
@@ -263,6 +267,13 @@ func expectedFSStats() kubestats.FsStats {
 	}
 }
 
+func expectedVolumeHealthStats() *kubestats.VolumeHealthStats {
+	metric := expectedMetrics()
+	return &kubestats.VolumeHealthStats{
+		Abnormal: *metric.Abnormal,
+	}
+}
+
 // Fake block-volume/metrics provider, block-devices have no inodes
 var _ volume.BlockVolume = &fakeBlockVolume{}
 
@@ -283,6 +294,7 @@ func expectedBlockMetrics() *volume.Metrics {
 		Available: resource.NewQuantity(available, resource.BinarySI),
 		Capacity:  resource.NewQuantity(capacity, resource.BinarySI),
 		Used:      resource.NewQuantity(available-capacity, resource.BinarySI),
+		Abnormal:  &volumeCondition.Abnormal,
 	}
 }
 

staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go

@@ -271,6 +271,17 @@ type VolumeStats struct {
 	// Reference to the PVC, if one exists
 	// +optional
 	PVCRef *PVCReference `json:"pvcRef,omitempty"`
+
+	// VolumeHealthStats contains data about volume health
+	// +optional
+	VolumeHealthStats *VolumeHealthStats `json:"volumeHealthStats,omitempty"`
+}
+
+// VolumeHealthStats contains data about volume health.
+type VolumeHealthStats struct {
+	// Normal volumes are available for use and operating optimally.
+	// An abnormal volume does not meet these criteria.
+	Abnormal bool `json:"abnormal"`
 }
 
 // PVCReference contains enough information to describe the referenced PVC.

test/e2e_node/summary_test.go

@@ -228,8 +228,9 @@ var _ = SIGDescribe("Summary API [NodeConformance]", func() {
 				}),
 				"VolumeStats": gstruct.MatchAllElements(summaryObjectID, gstruct.Elements{
 					"test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{
-						"Name":   gomega.Equal("test-empty-dir"),
+						"Name":              gomega.Equal("test-empty-dir"),
-						"PVCRef": gomega.BeNil(),
+						"PVCRef":            gomega.BeNil(),
+						"VolumeHealthStats": gomega.BeNil(),
 						"FsStats": gstruct.MatchAllFields(gstruct.Fields{
 							"Time":           recent(maxStatsAge),
 							"AvailableBytes": fsCapacityBounds,