github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-19 09:52:49 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #115965 from jsafrane/add-reconstruction-metrics

Browse Source 
Add volume reconstruction metrics
This commit is contained in:
Kubernetes Prow Robot 2023-03-06 14:56:16 -08:00 committed by GitHub
commit 44909771d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 50 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
pkg/kubelet/volumemanager/metrics/metrics.go
View File

@ -30,7 +30,11 @@ const (
pluginNameNotAvailable = "N/A"
// Metric keys for Volume Manager.
volumeManagerTotalVolumes = "volume_manager_total_volumes"
volumeManagerTotalVolumes = "volume_manager_total_volumes"
reconstructVolumeOperationsTotal = "reconstruct_volume_operations_total"
reconstructVolumeOperationsErrorsTotal = "reconstruct_volume_operations_errors_total"
forceCleanedFailedVolumeOperationsTotal = "force_cleaned_failed_volume_operations_total"
forceCleanedFailedVolumeOperationsErrorsTotal = "force_cleaned_failed_volume_operation_errors_total"
)
var (
@ -43,6 +47,36 @@ var (
nil,
metrics.ALPHA, "",
)
ReconstructVolumeOperationsTotal = metrics.NewCounter(
&metrics.CounterOpts{
Name: reconstructVolumeOperationsTotal,
Help: "The number of volumes that were attempted to be reconstructed from the operating system during kubelet startup. This includes both successful and failed reconstruction.",
StabilityLevel: metrics.ALPHA,
},
)
ReconstructVolumeOperationsErrorsTotal = metrics.NewCounter(
&metrics.CounterOpts{
Name: reconstructVolumeOperationsErrorsTotal,
Help: "The number of volumes that failed reconstruction from the operating system during kubelet startup.",
StabilityLevel: metrics.ALPHA,
},
)
ForceCleanedFailedVolumeOperationsTotal = metrics.NewCounter(
&metrics.CounterOpts{
Name: forceCleanedFailedVolumeOperationsTotal,
Help: "The number of volumes that were force cleaned after their reconstruction failed during kubelet startup. This includes both successful and failed cleanups.",
StabilityLevel: metrics.ALPHA,
},
)
ForceCleanedFailedVolumeOperationsErrorsTotal = metrics.NewCounter(
&metrics.CounterOpts{
Name: forceCleanedFailedVolumeOperationsErrorsTotal,
Help: "The number of volumes that failed force cleanup after their reconstruction failed during kubelet startup.",
StabilityLevel: metrics.ALPHA,
},
)
)
// volumeCount is a map of maps used as a counter.
@ -61,6 +95,10 @@ func (v volumeCount) add(state, plugin string) {
func Register(asw cache.ActualStateOfWorld, dsw cache.DesiredStateOfWorld, pluginMgr *volume.VolumePluginMgr) {
registerMetrics.Do(func() {
legacyregistry.CustomMustRegister(&totalVolumesCollector{asw: asw, dsw: dsw, pluginMgr: pluginMgr})
legacyregistry.MustRegister(ReconstructVolumeOperationsTotal)
legacyregistry.MustRegister(ReconstructVolumeOperationsErrorsTotal)
legacyregistry.MustRegister(ForceCleanedFailedVolumeOperationsTotal)
legacyregistry.MustRegister(ForceCleanedFailedVolumeOperationsErrorsTotal)
})
}

12
pkg/kubelet/volumemanager/reconciler/reconstruct_common.go
View File

@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/kubelet/config"
"k8s.io/kubernetes/pkg/kubelet/volumemanager/metrics"
volumepkg "k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
@ -94,10 +95,12 @@ func (rc *reconciler) cleanupMounts(volume podVolume) {
PluginName: volume.pluginName,
PodUID: types.UID(volume.podName),
}
metrics.ForceCleanedFailedVolumeOperationsTotal.Inc()
// TODO: Currently cleanupMounts only includes UnmountVolume operation. In the next PR, we will add
// to unmount both volume and device in the same routine.
err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
if err != nil {
metrics.ForceCleanedFailedVolumeOperationsErrorsTotal.Inc()
klog.ErrorS(err, mountedVolume.GenerateErrorDetailed("volumeHandler.UnmountVolumeHandler for UnmountVolume failed", err).Error())
return
}
@ -177,7 +180,14 @@ func getVolumesFromPodDir(podDir string) ([]podVolume, error) {
}
// Reconstruct volume data structure by reading the pod's volume directories
func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, error) {
func (rc *reconciler) reconstructVolume(volume podVolume) (rvolume *reconstructedVolume, rerr error) {
metrics.ReconstructVolumeOperationsTotal.Inc()
defer func() {
if rerr != nil {
metrics.ReconstructVolumeOperationsErrorsTotal.Inc()
}
}()
// plugin initializations
plugin, err := rc.volumePluginMgr.FindPluginByName(volume.pluginName)
if err != nil {