github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-28 05:57:25 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #65065 from sjenning/reduce-backoff-logging

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 66177, 66185, 67136, 67157, 65065). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubelet: reduce logging for backoff situations

xref https://bugzilla.redhat.com/show_bug.cgi?id=1555057#c6

Pods that are in `ImagePullBackOff` or `CrashLoopBackOff` currently generate a lot of logging at the `glog.Info()` level.  This PR moves some of that logging to `V(3)` and avoids logging in situations where the `SyncPod` only fails because pod are in a BackOff error condition.

@derekwaynecarr @liggitt
This commit is contained in:
Kubernetes Submit Queue 2018-08-15 02:09:20 -07:00 committed by GitHub
commit c65f65cf6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/kubelet/kubelet.go
View File

@ -1597,9 +1597,16 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error {
result := kl.containerRuntime.SyncPod(pod, apiPodStatus, podStatus, pullSecrets, kl.backOff) result := kl.containerRuntime.SyncPod(pod, apiPodStatus, podStatus, pullSecrets, kl.backOff)
kl.reasonCache.Update(pod.UID, result) kl.reasonCache.Update(pod.UID, result)
if err := result.Error(); err != nil { if err := result.Error(); err != nil {
// Do not record an event here, as we keep all event logging for sync pod failures // Do not return error if the only failures were pods in backoff
// local to container runtime so we get better errors for _, r := range result.SyncResults {
return err if r.Error != kubecontainer.ErrCrashLoopBackOff && r.Error != images.ErrImagePullBackOff {
// Do not record an event here, as we keep all event logging for sync pod failures
// local to container runtime so we get better errors
return err
}
}
return nil
} }
return nil return nil

6
pkg/kubelet/kuberuntime/kuberuntime_manager.go
View File

@ -510,7 +510,7 @@ func (m *kubeGenericRuntimeManager) computePodActions(pod *v1.Pod, podStatus *ku
if containerStatus == nil || containerStatus.State != kubecontainer.ContainerStateRunning { if containerStatus == nil || containerStatus.State != kubecontainer.ContainerStateRunning {
if kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) { if kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) {
message := fmt.Sprintf("Container %+v is dead, but RestartPolicy says that we should restart it.", container) message := fmt.Sprintf("Container %+v is dead, but RestartPolicy says that we should restart it.", container)
glog.Info(message) glog.V(3).Infof(message)
changes.ContainersToStart = append(changes.ContainersToStart, idx) changes.ContainersToStart = append(changes.ContainersToStart, idx)
} }
continue continue
@ -754,7 +754,7 @@ func (m *kubeGenericRuntimeManager) doBackOff(pod *v1.Pod, container *v1.Contain
return false, "", nil return false, "", nil
} }
glog.Infof("checking backoff for container %q in pod %q", container.Name, format.Pod(pod)) glog.V(3).Infof("checking backoff for container %q in pod %q", container.Name, format.Pod(pod))
// Use the finished time of the latest exited container as the start point to calculate whether to do back-off. // Use the finished time of the latest exited container as the start point to calculate whether to do back-off.
ts := cStatus.FinishedAt ts := cStatus.FinishedAt
// backOff requires a unique key to identify the container. // backOff requires a unique key to identify the container.
@ -764,7 +764,7 @@ func (m *kubeGenericRuntimeManager) doBackOff(pod *v1.Pod, container *v1.Contain
m.recorder.Eventf(ref, v1.EventTypeWarning, events.BackOffStartContainer, "Back-off restarting failed container") m.recorder.Eventf(ref, v1.EventTypeWarning, events.BackOffStartContainer, "Back-off restarting failed container")
} }
err := fmt.Errorf("Back-off %s restarting failed container=%s pod=%s", backOff.Get(key), container.Name, format.Pod(pod)) err := fmt.Errorf("Back-off %s restarting failed container=%s pod=%s", backOff.Get(key), container.Name, format.Pod(pod))
glog.Infof("%s", err.Error()) glog.V(3).Infof("%s", err.Error())
return true, err.Error(), kubecontainer.ErrCrashLoopBackOff return true, err.Error(), kubecontainer.ErrCrashLoopBackOff
} }