add gc metrics and collect sync errors

This commit is contained in:
atiratree
2021-12-06 23:21:13 +01:00
committed by Filip Křepinský
parent 2a2e3db782
commit 2fc401d0a2
2 changed files with 52 additions and 0 deletions

View File

@@ -46,6 +46,7 @@ import (
"k8s.io/klog/v2"
c "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/apis/config/scheme"
"k8s.io/kubernetes/pkg/controller/garbagecollector/metrics"
// import known versions
_ "k8s.io/client-go/kubernetes"
@@ -123,6 +124,8 @@ func NewGarbageCollector(
ignoredResources: ignoredResources,
}
metrics.Register()
return gc, nil
}
@@ -179,6 +182,7 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterf
// This can occur if there is an internal error in GetDeletableResources.
if len(newResources) == 0 {
klog.V(2).Infof("no resources reported by discovery, skipping garbage collector sync")
metrics.GarbageCollectorResourcesSyncError.Inc()
return
}
@@ -203,6 +207,7 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterf
newResources = GetDeletableResources(discoveryClient)
if len(newResources) == 0 {
klog.V(2).Infof("no resources reported by discovery (attempt %d)", attempt)
metrics.GarbageCollectorResourcesSyncError.Inc()
return false, nil
}
}
@@ -226,6 +231,7 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterf
// attempt.
if err := gc.resyncMonitors(newResources); err != nil {
utilruntime.HandleError(fmt.Errorf("failed to sync resource monitors (attempt %d): %v", attempt, err))
metrics.GarbageCollectorResourcesSyncError.Inc()
return false, nil
}
klog.V(4).Infof("resynced monitors")
@@ -237,6 +243,7 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterf
// note that workers stay paused until we successfully resync.
if !cache.WaitForNamedCacheSync("garbage collector", waitForStopOrTimeout(stopCh, period), gc.dependencyGraphBuilder.IsSynced) {
utilruntime.HandleError(fmt.Errorf("timed out waiting for dependency graph builder sync during GC sync (attempt %d)", attempt))
metrics.GarbageCollectorResourcesSyncError.Inc()
return false, nil
}