github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-28 05:57:25 +00:00
Code Issues Projects Releases Wiki Activity

Issue 71614: Protect log message maps

Browse Source
This commit is contained in:
Robert Krawitz 2018-11-30 15:49:35 -05:00
parent 3156df3e80
commit bc091be66a
2 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/kubelet/kuberuntime/kuberuntime_manager.go
View File

@ -20,6 +20,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"os" "os"
"sync"
"time" "time"
cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapi "github.com/google/cadvisor/info/v1"
@ -131,6 +132,7 @@ type kubeGenericRuntimeManager struct {
// Time last per-container error message was printed // Time last per-container error message was printed
errorPrinted map[string]time.Time errorPrinted map[string]time.Time
errorMapLock sync.Mutex
} }
// KubeGenericRuntime is a interface contains interfaces for container runtime and command. // KubeGenericRuntime is a interface contains interfaces for container runtime and command.
@ -830,6 +832,8 @@ func (m *kubeGenericRuntimeManager) killPodWithSyncResult(pod *v1.Pod, runningPo
} }
func (m *kubeGenericRuntimeManager) cleanupErrorTimeouts() { func (m *kubeGenericRuntimeManager) cleanupErrorTimeouts() {
m.errorMapLock.Lock()
defer m.errorMapLock.Unlock()
for name, timeout := range m.errorPrinted { for name, timeout := range m.errorPrinted {
if time.Now().Sub(timeout) >= identicalErrorDelay { if time.Now().Sub(timeout) >= identicalErrorDelay {
delete(m.errorPrinted, name) delete(m.errorPrinted, name)
@ -886,6 +890,8 @@ func (m *kubeGenericRuntimeManager) GetPodStatus(uid kubetypes.UID, name, namesp
// Get statuses of all containers visible in the pod. // Get statuses of all containers visible in the pod.
containerStatuses, err := m.getPodContainerStatuses(uid, name, namespace) containerStatuses, err := m.getPodContainerStatuses(uid, name, namespace)
m.errorMapLock.Lock()
defer m.errorMapLock.Unlock()
if err != nil { if err != nil {
lastMsg, ok := m.lastError[podFullName] lastMsg, ok := m.lastError[podFullName]
if !ok || err.Error() != lastMsg || time.Now().Sub(m.errorPrinted[podFullName]) >= identicalErrorDelay { if !ok || err.Error() != lastMsg || time.Now().Sub(m.errorPrinted[podFullName]) >= identicalErrorDelay {

12
pkg/kubelet/remote/remote_runtime.go
View File

@ -21,6 +21,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"sync"
"time" "time"
"google.golang.org/grpc" "google.golang.org/grpc"
@ -40,6 +41,7 @@ type RemoteRuntimeService struct {
lastError map[string]string lastError map[string]string
// Time last per-container error message was printed // Time last per-container error message was printed
errorPrinted map[string]time.Time errorPrinted map[string]time.Time
errorMapLock sync.Mutex
} }
const ( const (
@ -236,8 +238,10 @@ func (r *RemoteRuntimeService) StopContainer(containerID string, timeout int64)
ctx, cancel := getContextWithTimeout(t) ctx, cancel := getContextWithTimeout(t)
defer cancel() defer cancel()
r.errorMapLock.Lock()
delete(r.lastError, containerID) delete(r.lastError, containerID)
delete(r.errorPrinted, containerID) delete(r.errorPrinted, containerID)
r.errorMapLock.Unlock()
_, err := r.runtimeClient.StopContainer(ctx, &runtimeapi.StopContainerRequest{ _, err := r.runtimeClient.StopContainer(ctx, &runtimeapi.StopContainerRequest{
ContainerId: containerID, ContainerId: containerID,
Timeout: timeout, Timeout: timeout,
@ -256,8 +260,10 @@ func (r *RemoteRuntimeService) RemoveContainer(containerID string) error {
ctx, cancel := getContextWithTimeout(r.timeout) ctx, cancel := getContextWithTimeout(r.timeout)
defer cancel() defer cancel()
r.errorMapLock.Lock()
delete(r.lastError, containerID) delete(r.lastError, containerID)
delete(r.errorPrinted, containerID) delete(r.errorPrinted, containerID)
r.errorMapLock.Unlock()
_, err := r.runtimeClient.RemoveContainer(ctx, &runtimeapi.RemoveContainerRequest{ _, err := r.runtimeClient.RemoveContainer(ctx, &runtimeapi.RemoveContainerRequest{
ContainerId: containerID, ContainerId: containerID,
}) })
@ -287,6 +293,8 @@ func (r *RemoteRuntimeService) ListContainers(filter *runtimeapi.ContainerFilter
// Clean up any expired last-error timers // Clean up any expired last-error timers
func (r *RemoteRuntimeService) cleanupErrorTimeouts() { func (r *RemoteRuntimeService) cleanupErrorTimeouts() {
r.errorMapLock.Lock()
defer r.errorMapLock.Unlock()
for ID, timeout := range r.errorPrinted { for ID, timeout := range r.errorPrinted {
if time.Now().Sub(timeout) >= identicalErrorDelay { if time.Now().Sub(timeout) >= identicalErrorDelay {
delete(r.lastError, ID) delete(r.lastError, ID)
@ -304,6 +312,8 @@ func (r *RemoteRuntimeService) ContainerStatus(containerID string) (*runtimeapi.
ContainerId: containerID, ContainerId: containerID,
}) })
r.cleanupErrorTimeouts() r.cleanupErrorTimeouts()
r.errorMapLock.Lock()
defer r.errorMapLock.Unlock()
if err != nil { if err != nil {
// Don't spam the log with endless messages about the same failure. // Don't spam the log with endless messages about the same failure.
lastMsg, ok := r.lastError[containerID] lastMsg, ok := r.lastError[containerID]
@ -491,6 +501,8 @@ func (r *RemoteRuntimeService) ContainerStats(containerID string) (*runtimeapi.C
ContainerId: containerID, ContainerId: containerID,
}) })
r.cleanupErrorTimeouts() r.cleanupErrorTimeouts()
r.errorMapLock.Lock()
defer r.errorMapLock.Unlock()
if err != nil { if err != nil {
lastMsg, ok := r.lastError[containerID] lastMsg, ok := r.lastError[containerID]
if !ok || err.Error() != lastMsg || time.Now().Sub(r.errorPrinted[containerID]) >= identicalErrorDelay { if !ok || err.Error() != lastMsg || time.Now().Sub(r.errorPrinted[containerID]) >= identicalErrorDelay {