github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-29 14:37:00 +00:00
Code Issues Projects Releases Wiki Activity

Add metrics for force cleaned mounts after failed reconstruction

Browse Source 
Count nr. of force cleaned mounts + their failures after a volume fails
reconstruction.
This commit is contained in:
Jan Safranek 2023-02-22 13:00:35 +01:00
parent bd73aee9db
commit 9ca548fcf0
2 changed files with 32 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
pkg/kubelet/volumemanager/metrics/metrics.go
View File

@ -30,9 +30,11 @@ const (
pluginNameNotAvailable = "N/A"
// Metric keys for Volume Manager.
volumeManagerTotalVolumes = "volume_manager_total_volumes"
reconstructedVolumesTotal = "reconstructed_volumes_total"
reconstructedVolumesErrorsTotal = "reconstructed_volumes_errors_total"
volumeManagerTotalVolumes = "volume_manager_total_volumes"
reconstructVolumeOperationsTotal = "reconstruct_volume_operations_total"
reconstructVolumeOperationsErrorsTotal = "reconstruct_volume_operations_errors_total"
forceCleanedFailedVolumeOperationsTotal = "force_cleaned_failed_volume_operations_total"
forceCleanedFailedVolumeOperationsErrorsTotal = "force_cleaned_failed_volume_operation_errors_total"
)
var (
@ -46,20 +48,35 @@ var (
metrics.ALPHA, "",
)
ReconstructedVolumesTotal = metrics.NewCounter(
ReconstructVolumeOperationsTotal = metrics.NewCounter(
&metrics.CounterOpts{
Name: reconstructedVolumesTotal,
Name: reconstructVolumeOperationsTotal,
Help: "The number of volumes that were attempted to be reconstructed from the operating system during kubelet startup. This includes both successful and failed reconstruction.",
StabilityLevel: metrics.ALPHA,
},
)
ReconstructedVolumesErrorsTotal = metrics.NewCounter(
ReconstructVolumeOperationsErrorsTotal = metrics.NewCounter(
&metrics.CounterOpts{
Name: reconstructedVolumesErrorsTotal,
Name: reconstructVolumeOperationsErrorsTotal,
Help: "The number of volumes that failed reconstruction from the operating system during kubelet startup.",
StabilityLevel: metrics.ALPHA,
},
)
ForceCleanedFailedVolumeOperationsTotal = metrics.NewCounter(
&metrics.CounterOpts{
Name: forceCleanedFailedVolumeOperationsTotal,
Help: "The number of volumes that were force cleaned after their reconstruction failed during kubelet startup. This includes both successful and failed cleanups.",
StabilityLevel: metrics.ALPHA,
},
)
ForceCleanedFailedVolumeOperationsErrorsTotal = metrics.NewCounter(
&metrics.CounterOpts{
Name: forceCleanedFailedVolumeOperationsErrorsTotal,
Help: "The number of volumes that failed force cleanup after their reconstruction failed during kubelet startup.",
StabilityLevel: metrics.ALPHA,
},
)
)
// volumeCount is a map of maps used as a counter.
@ -78,8 +95,10 @@ func (v volumeCount) add(state, plugin string) {
func Register(asw cache.ActualStateOfWorld, dsw cache.DesiredStateOfWorld, pluginMgr *volume.VolumePluginMgr) {
registerMetrics.Do(func() {
legacyregistry.CustomMustRegister(&totalVolumesCollector{asw: asw, dsw: dsw, pluginMgr: pluginMgr})
legacyregistry.MustRegister(ReconstructedVolumesTotal)
legacyregistry.MustRegister(ReconstructedVolumesErrorsTotal)
legacyregistry.MustRegister(ReconstructVolumeOperationsTotal)
legacyregistry.MustRegister(ReconstructVolumeOperationsErrorsTotal)
legacyregistry.MustRegister(ForceCleanedFailedVolumeOperationsTotal)
legacyregistry.MustRegister(ForceCleanedFailedVolumeOperationsErrorsTotal)
})
}

6
pkg/kubelet/volumemanager/reconciler/reconstruct_common.go
View File

@ -95,10 +95,12 @@ func (rc *reconciler) cleanupMounts(volume podVolume) {
PluginName: volume.pluginName,
PodUID: types.UID(volume.podName),
}
metrics.ForceCleanedFailedVolumeOperationsTotal.Inc()
// TODO: Currently cleanupMounts only includes UnmountVolume operation. In the next PR, we will add
// to unmount both volume and device in the same routine.
err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
if err != nil {
metrics.ForceCleanedFailedVolumeOperationsErrorsTotal.Inc()
klog.ErrorS(err, mountedVolume.GenerateErrorDetailed("volumeHandler.UnmountVolumeHandler for UnmountVolume failed", err).Error())
return
}
@ -179,10 +181,10 @@ func getVolumesFromPodDir(podDir string) ([]podVolume, error) {
// Reconstruct volume data structure by reading the pod's volume directories
func (rc *reconciler) reconstructVolume(volume podVolume) (rvolume *reconstructedVolume, rerr error) {
metrics.ReconstructedVolumesTotal.Inc()
metrics.ReconstructVolumeOperationsTotal.Inc()
defer func() {
if rerr != nil {
metrics.ReconstructedVolumesErrorsTotal.Inc()
metrics.ReconstructVolumeOperationsErrorsTotal.Inc()
}
}()