From d71e21e01eca04b137b452bbcd6147a050a2f690 Mon Sep 17 00:00:00 2001 From: fengzixu Date: Sat, 9 Oct 2021 15:37:43 +0900 Subject: [PATCH 01/13] add volume kubelet_volume_stats_health_abnormal to kubelet --- pkg/kubelet/metrics/collectors/volume_stats.go | 16 ++++++++++++++++ .../metrics/collectors/volume_stats_test.go | 10 ++++++++++ pkg/kubelet/metrics/metrics.go | 1 + .../server/stats/volume_stat_calculator.go | 3 +++ .../server/stats/volume_stat_calculator_test.go | 1 + .../kubelet/pkg/apis/stats/v1alpha1/types.go | 11 +++++++++++ 6 files changed, 42 insertions(+) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index 8ff14a0b2ac..2fed2f3d0aa 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -61,6 +61,12 @@ var ( []string{"namespace", "persistentvolumeclaim"}, nil, metrics.ALPHA, "", ) + + volumeStatsHealthAbnormalDesc = metrics.NewDesc( + metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthAbnormalKey), + "Volume health status. The count is either 1 or 0", + []string{"namespace", "persistentvolumeclaim"}, nil, + metrics.ALPHA, "") ) type volumeStatsCollector struct { @@ -85,6 +91,7 @@ func (collector *volumeStatsCollector) DescribeWithStability(ch chan<- *metrics. ch <- volumeStatsInodesDesc ch <- volumeStatsInodesFreeDesc ch <- volumeStatsInodesUsedDesc + ch <- volumeStatsHealthAbnormalDesc } // CollectWithStability implements the metrics.StableCollector interface. @@ -120,7 +127,16 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me addGauge(volumeStatsInodesDesc, pvcRef, float64(*volumeStat.Inodes)) addGauge(volumeStatsInodesFreeDesc, pvcRef, float64(*volumeStat.InodesFree)) addGauge(volumeStatsInodesUsedDesc, pvcRef, float64(*volumeStat.InodesUsed)) + addGauge(volumeStatsHealthAbnormalDesc, pvcRef, float64(convertBoolToFloat64(volumeStat.Abnormal))) allPVCs.Insert(pvcUniqStr) } } } + +func convertBoolToFloat64(boolVal bool) float64 { + if boolVal { + return 1 + } + + return 0 +} diff --git a/pkg/kubelet/metrics/collectors/volume_stats_test.go b/pkg/kubelet/metrics/collectors/volume_stats_test.go index 54f07178c6d..2cbeff4e37b 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats_test.go +++ b/pkg/kubelet/metrics/collectors/volume_stats_test.go @@ -47,6 +47,8 @@ func TestVolumeStatsCollector(t *testing.T) { # TYPE kubelet_volume_stats_inodes_used gauge # HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume # TYPE kubelet_volume_stats_used_bytes gauge + # HELP kubelet_volume_stats_health_abnormal [ALPHA] Volume health status. The count is either 1 or 0 + # TYPE kubelet_volume_stats_health_abnormal gauge ` var ( @@ -83,6 +85,9 @@ func TestVolumeStatsCollector(t *testing.T) { Name: "testpvc", Namespace: "testns", }, + VolumeHealthStats: statsapi.VolumeHealthStats{ + Abnormal: true, + }, }, }, }, @@ -106,6 +111,9 @@ func TestVolumeStatsCollector(t *testing.T) { Name: "testpvc", Namespace: "testns", }, + VolumeHealthStats: statsapi.VolumeHealthStats{ + Abnormal: true, + }, }, }, }, @@ -118,6 +126,7 @@ func TestVolumeStatsCollector(t *testing.T) { kubelet_volume_stats_inodes_free{namespace="testns",persistentvolumeclaim="testpvc"} 655344 kubelet_volume_stats_inodes_used{namespace="testns",persistentvolumeclaim="testpvc"} 16 kubelet_volume_stats_used_bytes{namespace="testns",persistentvolumeclaim="testpvc"} 4.21789696e+09 + kubelet_volume_stats_health_abnormal{namespace="testns",persistentvolumeclaim="testpvc"} 1 ` metrics = []string{ @@ -127,6 +136,7 @@ func TestVolumeStatsCollector(t *testing.T) { "kubelet_volume_stats_inodes_free", "kubelet_volume_stats_inodes_used", "kubelet_volume_stats_used_bytes", + "kubelet_volume_stats_health_abnormal", } ) diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 302db0de4da..7187e8d84d6 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -53,6 +53,7 @@ const ( VolumeStatsInodesKey = "volume_stats_inodes" VolumeStatsInodesFreeKey = "volume_stats_inodes_free" VolumeStatsInodesUsedKey = "volume_stats_inodes_used" + VolumeStatsHealthAbnormalKey = "volume_stats_health_abnormal" RunningPodsKey = "running_pods" RunningContainersKey = "running_containers" // Metrics keys of remote runtime operations diff --git a/pkg/kubelet/server/stats/volume_stat_calculator.go b/pkg/kubelet/server/stats/volume_stat_calculator.go index 63e9b1e7c39..b28b541dc72 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -200,6 +200,9 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats return stats.VolumeStats{ Name: podName, PVCRef: pvcRef, + VolumeHealthStats: stats.VolumeHealthStats{ + Abnormal: *metric.Abnormal, + }, FsStats: stats.FsStats{Time: metric.Time, AvailableBytes: &available, CapacityBytes: &capacity, UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed}, } diff --git a/pkg/kubelet/server/stats/volume_stat_calculator_test.go b/pkg/kubelet/server/stats/volume_stat_calculator_test.go index 1edd0c8d7df..a230d7a1dfe 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator_test.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator_test.go @@ -283,6 +283,7 @@ func expectedBlockMetrics() *volume.Metrics { Available: resource.NewQuantity(available, resource.BinarySI), Capacity: resource.NewQuantity(capacity, resource.BinarySI), Used: resource.NewQuantity(available-capacity, resource.BinarySI), + Abnormal: &volumeCondition.Abnormal, } } diff --git a/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go b/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go index 50622f5170e..7efcc4dc269 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go @@ -269,6 +269,17 @@ type VolumeStats struct { // Reference to the PVC, if one exists // +optional PVCRef *PVCReference `json:"pvcRef,omitempty"` + + // VolumeHealthStats contains data about volume health + // +optional + VolumeHealthStats `json:"volumeHealthStats,omitempty"` +} + +// VolumeHealthStats contains data about volume health. +type VolumeHealthStats struct { + // Normal volumes are available for use and operating optimally. + // An abnormal volume does not meet these criteria. + Abnormal bool `json:"abnormal,omitempty"` } // PVCReference contains enough information to describe the referenced PVC. From bab175527452152246bce67ffee0bcbaf85251a4 Mon Sep 17 00:00:00 2001 From: fengzixu Date: Sun, 24 Oct 2021 15:59:32 +0900 Subject: [PATCH 02/13] fix: correct metrics expression --- pkg/kubelet/metrics/collectors/volume_stats.go | 4 ++-- pkg/kubelet/metrics/collectors/volume_stats_test.go | 8 ++++---- pkg/kubelet/metrics/metrics.go | 2 +- test/e2e_node/summary_test.go | 3 +++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index 2fed2f3d0aa..e580bdb1516 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -63,7 +63,7 @@ var ( ) volumeStatsHealthAbnormalDesc = metrics.NewDesc( - metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthAbnormalKey), + metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusKey), "Volume health status. The count is either 1 or 0", []string{"namespace", "persistentvolumeclaim"}, nil, metrics.ALPHA, "") @@ -127,7 +127,7 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me addGauge(volumeStatsInodesDesc, pvcRef, float64(*volumeStat.Inodes)) addGauge(volumeStatsInodesFreeDesc, pvcRef, float64(*volumeStat.InodesFree)) addGauge(volumeStatsInodesUsedDesc, pvcRef, float64(*volumeStat.InodesUsed)) - addGauge(volumeStatsHealthAbnormalDesc, pvcRef, float64(convertBoolToFloat64(volumeStat.Abnormal))) + addGauge(volumeStatsHealthAbnormalDesc, pvcRef, convertBoolToFloat64(volumeStat.Abnormal)) allPVCs.Insert(pvcUniqStr) } } diff --git a/pkg/kubelet/metrics/collectors/volume_stats_test.go b/pkg/kubelet/metrics/collectors/volume_stats_test.go index 2cbeff4e37b..c9d9e1b89bb 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats_test.go +++ b/pkg/kubelet/metrics/collectors/volume_stats_test.go @@ -47,8 +47,8 @@ func TestVolumeStatsCollector(t *testing.T) { # TYPE kubelet_volume_stats_inodes_used gauge # HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume # TYPE kubelet_volume_stats_used_bytes gauge - # HELP kubelet_volume_stats_health_abnormal [ALPHA] Volume health status. The count is either 1 or 0 - # TYPE kubelet_volume_stats_health_abnormal gauge + # HELP kubelet_volume_stats_health_status [ALPHA] Volume health status. The count is either 1 or 0 + # TYPE kubelet_volume_stats_health_status gauge ` var ( @@ -126,7 +126,7 @@ func TestVolumeStatsCollector(t *testing.T) { kubelet_volume_stats_inodes_free{namespace="testns",persistentvolumeclaim="testpvc"} 655344 kubelet_volume_stats_inodes_used{namespace="testns",persistentvolumeclaim="testpvc"} 16 kubelet_volume_stats_used_bytes{namespace="testns",persistentvolumeclaim="testpvc"} 4.21789696e+09 - kubelet_volume_stats_health_abnormal{namespace="testns",persistentvolumeclaim="testpvc"} 1 + kubelet_volume_stats_health_status{namespace="testns",persistentvolumeclaim="testpvc"} 1 ` metrics = []string{ @@ -136,7 +136,7 @@ func TestVolumeStatsCollector(t *testing.T) { "kubelet_volume_stats_inodes_free", "kubelet_volume_stats_inodes_used", "kubelet_volume_stats_used_bytes", - "kubelet_volume_stats_health_abnormal", + "kubelet_volume_stats_health_status", } ) diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 7187e8d84d6..c0c31c6645a 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -53,7 +53,7 @@ const ( VolumeStatsInodesKey = "volume_stats_inodes" VolumeStatsInodesFreeKey = "volume_stats_inodes_free" VolumeStatsInodesUsedKey = "volume_stats_inodes_used" - VolumeStatsHealthAbnormalKey = "volume_stats_health_abnormal" + VolumeStatsHealthStatusKey = "volume_stats_health_status" RunningPodsKey = "running_pods" RunningContainersKey = "running_containers" // Metrics keys of remote runtime operations diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go index 704476a672c..cdb5302c38d 100644 --- a/test/e2e_node/summary_test.go +++ b/test/e2e_node/summary_test.go @@ -230,6 +230,9 @@ var _ = SIGDescribe("Summary API [NodeConformance]", func() { "test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{ "Name": gomega.Equal("test-empty-dir"), "PVCRef": gomega.BeNil(), + "VolumeHealthStats": gstruct.MatchAllFields(gstruct.Fields{ + "Abnormal": gomega.BeTrue(), + }), "FsStats": gstruct.MatchAllFields(gstruct.Fields{ "Time": recent(maxStatsAge), "AvailableBytes": fsCapacityBounds, From ed7fd0ced579b6feec7bde79b0981309020ef911 Mon Sep 17 00:00:00 2001 From: fengzixu Date: Fri, 26 Nov 2021 10:50:24 +0900 Subject: [PATCH 03/13] add volumeHealth label to metrics --- .../metrics/collectors/volume_stats.go | 5 ++--- .../metrics/collectors/volume_stats_test.go | 6 ++--- .../server/stats/volume_stat_calculator.go | 22 ++++++++++++------- .../stats/volume_stat_calculator_test.go | 20 ++++++++++++----- .../kubelet/pkg/apis/stats/v1alpha1/types.go | 4 ++-- test/e2e_node/summary_test.go | 8 ++++--- 6 files changed, 41 insertions(+), 24 deletions(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index e580bdb1516..3294150c2eb 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -65,7 +65,7 @@ var ( volumeStatsHealthAbnormalDesc = metrics.NewDesc( metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusKey), "Volume health status. The count is either 1 or 0", - []string{"namespace", "persistentvolumeclaim"}, nil, + []string{"namespace", "persistentvolumeclaim", "volume_health_status"}, nil, metrics.ALPHA, "") ) @@ -102,7 +102,6 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me } addGauge := func(desc *metrics.Desc, pvcRef *stats.PVCReference, v float64, lv ...string) { lv = append([]string{pvcRef.Namespace, pvcRef.Name}, lv...) - ch <- metrics.NewLazyConstMetric(desc, metrics.GaugeValue, v, lv...) } allPVCs := sets.String{} @@ -127,7 +126,7 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me addGauge(volumeStatsInodesDesc, pvcRef, float64(*volumeStat.Inodes)) addGauge(volumeStatsInodesFreeDesc, pvcRef, float64(*volumeStat.InodesFree)) addGauge(volumeStatsInodesUsedDesc, pvcRef, float64(*volumeStat.InodesUsed)) - addGauge(volumeStatsHealthAbnormalDesc, pvcRef, convertBoolToFloat64(volumeStat.Abnormal)) + addGauge(volumeStatsHealthAbnormalDesc, pvcRef, convertBoolToFloat64(volumeStat.VolumeHealthStats.Abnormal), "abnormal") allPVCs.Insert(pvcUniqStr) } } diff --git a/pkg/kubelet/metrics/collectors/volume_stats_test.go b/pkg/kubelet/metrics/collectors/volume_stats_test.go index c9d9e1b89bb..032b69337e1 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats_test.go +++ b/pkg/kubelet/metrics/collectors/volume_stats_test.go @@ -85,7 +85,7 @@ func TestVolumeStatsCollector(t *testing.T) { Name: "testpvc", Namespace: "testns", }, - VolumeHealthStats: statsapi.VolumeHealthStats{ + VolumeHealthStats: &statsapi.VolumeHealthStats{ Abnormal: true, }, }, @@ -111,7 +111,7 @@ func TestVolumeStatsCollector(t *testing.T) { Name: "testpvc", Namespace: "testns", }, - VolumeHealthStats: statsapi.VolumeHealthStats{ + VolumeHealthStats: &statsapi.VolumeHealthStats{ Abnormal: true, }, }, @@ -126,7 +126,7 @@ func TestVolumeStatsCollector(t *testing.T) { kubelet_volume_stats_inodes_free{namespace="testns",persistentvolumeclaim="testpvc"} 655344 kubelet_volume_stats_inodes_used{namespace="testns",persistentvolumeclaim="testpvc"} 16 kubelet_volume_stats_used_bytes{namespace="testns",persistentvolumeclaim="testpvc"} 4.21789696e+09 - kubelet_volume_stats_health_status{namespace="testns",persistentvolumeclaim="testpvc"} 1 + kubelet_volume_stats_health_status{namespace="testns",persistentvolumeclaim="testpvc",volume_health_status="abnormal"} 1 ` metrics = []string{ diff --git a/pkg/kubelet/server/stats/volume_stat_calculator.go b/pkg/kubelet/server/stats/volume_stat_calculator.go index b28b541dc72..d429e867303 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -177,7 +177,11 @@ func (s *volumeStatCalculator) calcAndStoreStats() { // parsePodVolumeStats converts (internal) volume.Metrics to (external) stats.VolumeStats structures func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats.PVCReference, metric *volume.Metrics, volSpec v1.Volume) stats.VolumeStats { - var available, capacity, used, inodes, inodesFree, inodesUsed uint64 + var ( + available, capacity, used, inodes, inodesFree, inodesUsed uint64 + volumeStats stats.VolumeStats + ) + if metric.Available != nil { available = uint64(metric.Available.Value()) } @@ -197,13 +201,15 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats inodesUsed = uint64(metric.InodesUsed.Value()) } - return stats.VolumeStats{ - Name: podName, - PVCRef: pvcRef, - VolumeHealthStats: stats.VolumeHealthStats{ + volumeStats.FsStats = stats.FsStats{Time: metric.Time, AvailableBytes: &available, CapacityBytes: &capacity, + UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed} + volumeStats.Name = podName + volumeStats.PVCRef = pvcRef + if metric.Abnormal != nil { + volumeStats.VolumeHealthStats = &stats.VolumeHealthStats{ Abnormal: *metric.Abnormal, - }, - FsStats: stats.FsStats{Time: metric.Time, AvailableBytes: &available, CapacityBytes: &capacity, - UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed}, + } } + + return volumeStats } diff --git a/pkg/kubelet/server/stats/volume_stat_calculator_test.go b/pkg/kubelet/server/stats/volume_stat_calculator_test.go index a230d7a1dfe..ed29b39b4c7 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator_test.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator_test.go @@ -128,8 +128,9 @@ func TestPVCRef(t *testing.T) { assert.Len(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), 4) // Verify 'vol0' doesn't have a PVC reference assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{ - Name: vol0, - FsStats: expectedFSStats(), + Name: vol0, + FsStats: expectedFSStats(), + VolumeHealthStats: expectedVolumeHealthStats(), }) // Verify 'vol1' has a PVC reference assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{ @@ -138,16 +139,18 @@ func TestPVCRef(t *testing.T) { Name: pvcClaimName0, Namespace: namespace0, }, - FsStats: expectedFSStats(), + FsStats: expectedFSStats(), + VolumeHealthStats: expectedVolumeHealthStats(), }) - // Verify 'vol2' has a PVC reference + // // Verify 'vol2' has a PVC reference assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{ Name: vol2, PVCRef: &kubestats.PVCReference{ Name: pvcClaimName1, Namespace: namespace0, }, - FsStats: expectedBlockStats(), + FsStats: expectedBlockStats(), + VolumeHealthStats: expectedVolumeHealthStats(), }) // Verify 'vol3' has a PVC reference assert.Contains(t, append(vs.EphemeralVolumes, vs.PersistentVolumes...), kubestats.VolumeStats{ @@ -263,6 +266,13 @@ func expectedFSStats() kubestats.FsStats { } } +func expectedVolumeHealthStats() *kubestats.VolumeHealthStats { + metric := expectedMetrics() + return &kubestats.VolumeHealthStats{ + Abnormal: *metric.Abnormal, + } +} + // Fake block-volume/metrics provider, block-devices have no inodes var _ volume.BlockVolume = &fakeBlockVolume{} diff --git a/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go b/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go index 7efcc4dc269..cf7797d3d2e 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go @@ -272,14 +272,14 @@ type VolumeStats struct { // VolumeHealthStats contains data about volume health // +optional - VolumeHealthStats `json:"volumeHealthStats,omitempty"` + VolumeHealthStats *VolumeHealthStats `json:"volumeHealthStats,omitempty"` } // VolumeHealthStats contains data about volume health. type VolumeHealthStats struct { // Normal volumes are available for use and operating optimally. // An abnormal volume does not meet these criteria. - Abnormal bool `json:"abnormal,omitempty"` + Abnormal bool `json:"abnormal"` } // PVCReference contains enough information to describe the referenced PVC. diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go index cdb5302c38d..8e85b38b1b3 100644 --- a/test/e2e_node/summary_test.go +++ b/test/e2e_node/summary_test.go @@ -230,9 +230,11 @@ var _ = SIGDescribe("Summary API [NodeConformance]", func() { "test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{ "Name": gomega.Equal("test-empty-dir"), "PVCRef": gomega.BeNil(), - "VolumeHealthStats": gstruct.MatchAllFields(gstruct.Fields{ - "Abnormal": gomega.BeTrue(), - }), + "VolumeHealthStats": gstruct.MatchAllFields( + gstruct.Fields{ + "Abnormal": gomega.BeFalse(), + }, + ), "FsStats": gstruct.MatchAllFields(gstruct.Fields{ "Time": recent(maxStatsAge), "AvailableBytes": fsCapacityBounds, From f202164c874da55ac84c794d9b9afa62dd01445f Mon Sep 17 00:00:00 2001 From: fengzixu Date: Fri, 26 Nov 2021 11:20:13 +0900 Subject: [PATCH 04/13] fix e2e test --- test/e2e_node/summary_test.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go index 8e85b38b1b3..dea6ed6f02d 100644 --- a/test/e2e_node/summary_test.go +++ b/test/e2e_node/summary_test.go @@ -228,13 +228,9 @@ var _ = SIGDescribe("Summary API [NodeConformance]", func() { }), "VolumeStats": gstruct.MatchAllElements(summaryObjectID, gstruct.Elements{ "test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{ - "Name": gomega.Equal("test-empty-dir"), - "PVCRef": gomega.BeNil(), - "VolumeHealthStats": gstruct.MatchAllFields( - gstruct.Fields{ - "Abnormal": gomega.BeFalse(), - }, - ), + "Name": gomega.Equal("test-empty-dir"), + "PVCRef": gomega.BeNil(), + "VolumeHealthStats": gomega.BeNil(), "FsStats": gstruct.MatchAllFields(gstruct.Fields{ "Time": recent(maxStatsAge), "AvailableBytes": fsCapacityBounds, From b885deffe31a59d04d7783cd465827202dbf0b42 Mon Sep 17 00:00:00 2001 From: fengzixu Date: Fri, 26 Nov 2021 13:17:49 +0900 Subject: [PATCH 05/13] fix unit test --- pkg/kubelet/server/stats/volume_stat_calculator_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/server/stats/volume_stat_calculator_test.go b/pkg/kubelet/server/stats/volume_stat_calculator_test.go index ed29b39b4c7..a1d3ce4898b 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator_test.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator_test.go @@ -159,7 +159,8 @@ func TestPVCRef(t *testing.T) { Name: pName0 + "-" + vol3, Namespace: namespace0, }, - FsStats: expectedFSStats(), + FsStats: expectedFSStats(), + VolumeHealthStats: expectedVolumeHealthStats(), }) } From 4a72f08a28583289b3826beecd32a357d9a9235a Mon Sep 17 00:00:00 2001 From: fengzixu Date: Mon, 29 Nov 2021 17:55:47 +0900 Subject: [PATCH 06/13] add useful comment for volume stats metrics --- pkg/kubelet/metrics/collectors/volume_stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index 3294150c2eb..be76c63ff8d 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -64,7 +64,7 @@ var ( volumeStatsHealthAbnormalDesc = metrics.NewDesc( metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusKey), - "Volume health status. The count is either 1 or 0", + "Volume health status. The value 1 of this metrics indicates volume is unhealthy, 0 indicates the unhealthy volume", []string{"namespace", "persistentvolumeclaim", "volume_health_status"}, nil, metrics.ALPHA, "") ) From 1cdc694ac2499641caa10e0f7ec6c24644593a0e Mon Sep 17 00:00:00 2001 From: fengzixu Date: Mon, 29 Nov 2021 21:49:54 +0900 Subject: [PATCH 07/13] fix unit test --- pkg/kubelet/metrics/collectors/volume_stats.go | 2 +- pkg/kubelet/metrics/collectors/volume_stats_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index be76c63ff8d..db175a9fc12 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -64,7 +64,7 @@ var ( volumeStatsHealthAbnormalDesc = metrics.NewDesc( metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusKey), - "Volume health status. The value 1 of this metrics indicates volume is unhealthy, 0 indicates the unhealthy volume", + "Volume health status. The value 1 of this metrics indicates unhealthy volume, 0 indicates the healthy", []string{"namespace", "persistentvolumeclaim", "volume_health_status"}, nil, metrics.ALPHA, "") ) diff --git a/pkg/kubelet/metrics/collectors/volume_stats_test.go b/pkg/kubelet/metrics/collectors/volume_stats_test.go index 032b69337e1..d07979c95ce 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats_test.go +++ b/pkg/kubelet/metrics/collectors/volume_stats_test.go @@ -47,7 +47,7 @@ func TestVolumeStatsCollector(t *testing.T) { # TYPE kubelet_volume_stats_inodes_used gauge # HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume # TYPE kubelet_volume_stats_used_bytes gauge - # HELP kubelet_volume_stats_health_status [ALPHA] Volume health status. The count is either 1 or 0 + # HELP kubelet_volume_stats_health_status [ALPHA] Volume health status. The value 1 of this metrics indicates unhealthy volume, 0 indicates the healthy # TYPE kubelet_volume_stats_health_status gauge ` From 5593e27429ff5a7c1bedf5fe3882e93c32af2f8a Mon Sep 17 00:00:00 2001 From: fengzixu Date: Tue, 30 Nov 2021 00:11:14 +0900 Subject: [PATCH 08/13] improve metrics comment --- pkg/kubelet/metrics/collectors/volume_stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index db175a9fc12..191c92f7c9e 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -64,7 +64,7 @@ var ( volumeStatsHealthAbnormalDesc = metrics.NewDesc( metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusKey), - "Volume health status. The value 1 of this metrics indicates unhealthy volume, 0 indicates the healthy", + "Volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy and 0 indicates volume is healthy", []string{"namespace", "persistentvolumeclaim", "volume_health_status"}, nil, metrics.ALPHA, "") ) From c1a58d715c98d2a7cb5aaa1a861430ebbb1c1aaa Mon Sep 17 00:00:00 2001 From: fengzixu Date: Tue, 30 Nov 2021 09:49:19 +0900 Subject: [PATCH 09/13] fix unit test --- pkg/kubelet/metrics/collectors/volume_stats_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats_test.go b/pkg/kubelet/metrics/collectors/volume_stats_test.go index d07979c95ce..50d33298bdc 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats_test.go +++ b/pkg/kubelet/metrics/collectors/volume_stats_test.go @@ -47,7 +47,7 @@ func TestVolumeStatsCollector(t *testing.T) { # TYPE kubelet_volume_stats_inodes_used gauge # HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume # TYPE kubelet_volume_stats_used_bytes gauge - # HELP kubelet_volume_stats_health_status [ALPHA] Volume health status. The value 1 of this metrics indicates unhealthy volume, 0 indicates the healthy + # HELP kubelet_volume_stats_health_status [ALPHA] Volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy and 0 indicates volume is healthy # TYPE kubelet_volume_stats_health_status gauge ` From e2b5b5465ae07a07abdb8343af1e2a588c7c4a7d Mon Sep 17 00:00:00 2001 From: fengzixu Date: Thu, 16 Dec 2021 21:36:38 +0900 Subject: [PATCH 10/13] improve metrics comment --- pkg/kubelet/metrics/collectors/volume_stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index 191c92f7c9e..5c96fcc54c1 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -64,7 +64,7 @@ var ( volumeStatsHealthAbnormalDesc = metrics.NewDesc( metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusKey), - "Volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy and 0 indicates volume is healthy", + "Volume health status. The count is either 1 or 0. When volume_health_status is abnormal, 1 indicates the volume is unhealthy, 0 indicates volume is healthy", []string{"namespace", "persistentvolumeclaim", "volume_health_status"}, nil, metrics.ALPHA, "") ) From f96449f2e2550e3f8119a23374c203baa7230ccf Mon Sep 17 00:00:00 2001 From: fengzixu Date: Thu, 16 Dec 2021 22:17:21 +0900 Subject: [PATCH 11/13] fix unit test --- pkg/kubelet/metrics/collectors/volume_stats_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats_test.go b/pkg/kubelet/metrics/collectors/volume_stats_test.go index 50d33298bdc..7d4d5ba1d68 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats_test.go +++ b/pkg/kubelet/metrics/collectors/volume_stats_test.go @@ -47,7 +47,7 @@ func TestVolumeStatsCollector(t *testing.T) { # TYPE kubelet_volume_stats_inodes_used gauge # HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume # TYPE kubelet_volume_stats_used_bytes gauge - # HELP kubelet_volume_stats_health_status [ALPHA] Volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy and 0 indicates volume is healthy + # HELP kubelet_volume_stats_health_status [ALPHA] Volume health status. The count is either 1 or 0. When volume_health_status is abnormal, 1 indicates the volume is unhealthy, 0 indicates volume is healthy # TYPE kubelet_volume_stats_health_status gauge ` From 5d544d3f0179e188b69f05220c6c00b0260ed170 Mon Sep 17 00:00:00 2001 From: fengzixu Date: Tue, 11 Jan 2022 14:28:31 +0000 Subject: [PATCH 12/13] fix comment --- pkg/kubelet/metrics/collectors/volume_stats.go | 2 +- pkg/kubelet/metrics/collectors/volume_stats_test.go | 2 +- pkg/kubelet/server/stats/volume_stat_calculator.go | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index 5c96fcc54c1..b20593dd66c 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -64,7 +64,7 @@ var ( volumeStatsHealthAbnormalDesc = metrics.NewDesc( metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusKey), - "Volume health status. The count is either 1 or 0. When volume_health_status is abnormal, 1 indicates the volume is unhealthy, 0 indicates volume is healthy", + "Abnormal volume health status. The count is either 1 or 0. When volume_health_status is abnormal, 1 indicates the volume is unhealthy, 0 indicates volume is healthy", []string{"namespace", "persistentvolumeclaim", "volume_health_status"}, nil, metrics.ALPHA, "") ) diff --git a/pkg/kubelet/metrics/collectors/volume_stats_test.go b/pkg/kubelet/metrics/collectors/volume_stats_test.go index 7d4d5ba1d68..66f94f61c0f 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats_test.go +++ b/pkg/kubelet/metrics/collectors/volume_stats_test.go @@ -47,7 +47,7 @@ func TestVolumeStatsCollector(t *testing.T) { # TYPE kubelet_volume_stats_inodes_used gauge # HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume # TYPE kubelet_volume_stats_used_bytes gauge - # HELP kubelet_volume_stats_health_status [ALPHA] Volume health status. The count is either 1 or 0. When volume_health_status is abnormal, 1 indicates the volume is unhealthy, 0 indicates volume is healthy + # HELP kubelet_volume_stats_health_status [ALPHA] Abnormal volume health status. The count is either 1 or 0. When volume_health_status is abnormal, 1 indicates the volume is unhealthy, 0 indicates volume is healthy # TYPE kubelet_volume_stats_health_status gauge ` diff --git a/pkg/kubelet/server/stats/volume_stat_calculator.go b/pkg/kubelet/server/stats/volume_stat_calculator.go index d429e867303..f2580e0e679 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -179,7 +179,6 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats var ( available, capacity, used, inodes, inodesFree, inodesUsed uint64 - volumeStats stats.VolumeStats ) if metric.Available != nil { @@ -201,10 +200,13 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats inodesUsed = uint64(metric.InodesUsed.Value()) } - volumeStats.FsStats = stats.FsStats{Time: metric.Time, AvailableBytes: &available, CapacityBytes: &capacity, - UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed} - volumeStats.Name = podName - volumeStats.PVCRef = pvcRef + volumeStats := stats.VolumeStats{ + Name: podName, + PVCRef: pvcRef, + FsStats: stats.FsStats{Time: metric.Time, AvailableBytes: &available, CapacityBytes: &capacity, + UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed}, + } + if metric.Abnormal != nil { volumeStats.VolumeHealthStats = &stats.VolumeHealthStats{ Abnormal: *metric.Abnormal, From 9808ae48a0a2aa46469f0262354dea127bf9746b Mon Sep 17 00:00:00 2001 From: fengzixu Date: Sun, 23 Jan 2022 02:44:10 +0000 Subject: [PATCH 13/13] change the volume health status metrics name --- .../metrics/collectors/volume_stats.go | 8 ++-- .../metrics/collectors/volume_stats_test.go | 8 ++-- pkg/kubelet/metrics/metrics.go | 46 +++++++++---------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/pkg/kubelet/metrics/collectors/volume_stats.go b/pkg/kubelet/metrics/collectors/volume_stats.go index b20593dd66c..f41a2f47043 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats.go +++ b/pkg/kubelet/metrics/collectors/volume_stats.go @@ -63,9 +63,9 @@ var ( ) volumeStatsHealthAbnormalDesc = metrics.NewDesc( - metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusKey), - "Abnormal volume health status. The count is either 1 or 0. When volume_health_status is abnormal, 1 indicates the volume is unhealthy, 0 indicates volume is healthy", - []string{"namespace", "persistentvolumeclaim", "volume_health_status"}, nil, + metrics.BuildFQName("", kubeletmetrics.KubeletSubsystem, kubeletmetrics.VolumeStatsHealthStatusAbnormalKey), + "Abnormal volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy, 0 indicates volume is healthy", + []string{"namespace", "persistentvolumeclaim"}, nil, metrics.ALPHA, "") ) @@ -126,7 +126,7 @@ func (collector *volumeStatsCollector) CollectWithStability(ch chan<- metrics.Me addGauge(volumeStatsInodesDesc, pvcRef, float64(*volumeStat.Inodes)) addGauge(volumeStatsInodesFreeDesc, pvcRef, float64(*volumeStat.InodesFree)) addGauge(volumeStatsInodesUsedDesc, pvcRef, float64(*volumeStat.InodesUsed)) - addGauge(volumeStatsHealthAbnormalDesc, pvcRef, convertBoolToFloat64(volumeStat.VolumeHealthStats.Abnormal), "abnormal") + addGauge(volumeStatsHealthAbnormalDesc, pvcRef, convertBoolToFloat64(volumeStat.VolumeHealthStats.Abnormal)) allPVCs.Insert(pvcUniqStr) } } diff --git a/pkg/kubelet/metrics/collectors/volume_stats_test.go b/pkg/kubelet/metrics/collectors/volume_stats_test.go index 66f94f61c0f..1599b39cd15 100644 --- a/pkg/kubelet/metrics/collectors/volume_stats_test.go +++ b/pkg/kubelet/metrics/collectors/volume_stats_test.go @@ -47,8 +47,8 @@ func TestVolumeStatsCollector(t *testing.T) { # TYPE kubelet_volume_stats_inodes_used gauge # HELP kubelet_volume_stats_used_bytes [ALPHA] Number of used bytes in the volume # TYPE kubelet_volume_stats_used_bytes gauge - # HELP kubelet_volume_stats_health_status [ALPHA] Abnormal volume health status. The count is either 1 or 0. When volume_health_status is abnormal, 1 indicates the volume is unhealthy, 0 indicates volume is healthy - # TYPE kubelet_volume_stats_health_status gauge + # HELP kubelet_volume_stats_health_status_abnormal [ALPHA] Abnormal volume health status. The count is either 1 or 0. 1 indicates the volume is unhealthy, 0 indicates volume is healthy + # TYPE kubelet_volume_stats_health_status_abnormal gauge ` var ( @@ -126,7 +126,7 @@ func TestVolumeStatsCollector(t *testing.T) { kubelet_volume_stats_inodes_free{namespace="testns",persistentvolumeclaim="testpvc"} 655344 kubelet_volume_stats_inodes_used{namespace="testns",persistentvolumeclaim="testpvc"} 16 kubelet_volume_stats_used_bytes{namespace="testns",persistentvolumeclaim="testpvc"} 4.21789696e+09 - kubelet_volume_stats_health_status{namespace="testns",persistentvolumeclaim="testpvc",volume_health_status="abnormal"} 1 + kubelet_volume_stats_health_status_abnormal{namespace="testns",persistentvolumeclaim="testpvc"} 1 ` metrics = []string{ @@ -136,7 +136,7 @@ func TestVolumeStatsCollector(t *testing.T) { "kubelet_volume_stats_inodes_free", "kubelet_volume_stats_inodes_used", "kubelet_volume_stats_used_bytes", - "kubelet_volume_stats_health_status", + "kubelet_volume_stats_health_status_abnormal", } ) diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index c0c31c6645a..a71903e2a62 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -33,29 +33,29 @@ import ( // This const block defines the metric names for the kubelet metrics. const ( - KubeletSubsystem = "kubelet" - NodeNameKey = "node_name" - NodeLabelKey = "node" - PodWorkerDurationKey = "pod_worker_duration_seconds" - PodStartDurationKey = "pod_start_duration_seconds" - CgroupManagerOperationsKey = "cgroup_manager_duration_seconds" - PodWorkerStartDurationKey = "pod_worker_start_duration_seconds" - PLEGRelistDurationKey = "pleg_relist_duration_seconds" - PLEGDiscardEventsKey = "pleg_discard_events" - PLEGRelistIntervalKey = "pleg_relist_interval_seconds" - PLEGLastSeenKey = "pleg_last_seen_seconds" - EvictionsKey = "evictions" - EvictionStatsAgeKey = "eviction_stats_age_seconds" - PreemptionsKey = "preemptions" - VolumeStatsCapacityBytesKey = "volume_stats_capacity_bytes" - VolumeStatsAvailableBytesKey = "volume_stats_available_bytes" - VolumeStatsUsedBytesKey = "volume_stats_used_bytes" - VolumeStatsInodesKey = "volume_stats_inodes" - VolumeStatsInodesFreeKey = "volume_stats_inodes_free" - VolumeStatsInodesUsedKey = "volume_stats_inodes_used" - VolumeStatsHealthStatusKey = "volume_stats_health_status" - RunningPodsKey = "running_pods" - RunningContainersKey = "running_containers" + KubeletSubsystem = "kubelet" + NodeNameKey = "node_name" + NodeLabelKey = "node" + PodWorkerDurationKey = "pod_worker_duration_seconds" + PodStartDurationKey = "pod_start_duration_seconds" + CgroupManagerOperationsKey = "cgroup_manager_duration_seconds" + PodWorkerStartDurationKey = "pod_worker_start_duration_seconds" + PLEGRelistDurationKey = "pleg_relist_duration_seconds" + PLEGDiscardEventsKey = "pleg_discard_events" + PLEGRelistIntervalKey = "pleg_relist_interval_seconds" + PLEGLastSeenKey = "pleg_last_seen_seconds" + EvictionsKey = "evictions" + EvictionStatsAgeKey = "eviction_stats_age_seconds" + PreemptionsKey = "preemptions" + VolumeStatsCapacityBytesKey = "volume_stats_capacity_bytes" + VolumeStatsAvailableBytesKey = "volume_stats_available_bytes" + VolumeStatsUsedBytesKey = "volume_stats_used_bytes" + VolumeStatsInodesKey = "volume_stats_inodes" + VolumeStatsInodesFreeKey = "volume_stats_inodes_free" + VolumeStatsInodesUsedKey = "volume_stats_inodes_used" + VolumeStatsHealthStatusAbnormalKey = "volume_stats_health_status_abnormal" + RunningPodsKey = "running_pods" + RunningContainersKey = "running_containers" // Metrics keys of remote runtime operations RuntimeOperationsKey = "runtime_operations_total" RuntimeOperationsDurationKey = "runtime_operations_duration_seconds"