Merge pull request #42916 from dashpole/misleading_log

Automatic merge from submit-queue

Clearer ImageGC failure errors.  Fewer events.

Addresses #26000.  Kubelet often "fails" image garbage collection if cAdvisor has not completed the first round of stats collection.  Don't create events for a single failure, and make log messages more specific.

@kubernetes/sig-node-bugs
This commit is contained in:
Kubernetes Submit Queue 2017-04-04 11:23:32 -07:00 committed by GitHub
commit faf2eca226

View File

@ -1159,17 +1159,22 @@ func (kl *Kubelet) StartGarbageCollection() {
}
}, ContainerGCPeriod, wait.NeverStop)
loggedImageGCFailure := false
prevImageGCFailed := false
go wait.Until(func() {
if err := kl.imageManager.GarbageCollect(); err != nil {
glog.Errorf("Image garbage collection failed: %v", err)
kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.ImageGCFailed, err.Error())
loggedImageGCFailure = true
if prevImageGCFailed {
glog.Errorf("Image garbage collection failed multiple times in a row: %v", err)
// Only create an event for repeated failures
kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.ImageGCFailed, err.Error())
} else {
glog.Errorf("Image garbage collection failed once. Stats initialization may not have completed yet: %v", err)
}
prevImageGCFailed = true
} else {
var vLevel glog.Level = 4
if loggedImageGCFailure {
if prevImageGCFailed {
vLevel = 1
loggedImageGCFailure = false
prevImageGCFailed = false
}
glog.V(vLevel).Infof("Image garbage collection succeeded")