From fe089a2d12cdb70aa07b3613b81bad754ab52a74 Mon Sep 17 00:00:00 2001 From: choury Date: Wed, 20 Jan 2021 10:26:42 +0800 Subject: [PATCH] make podTopologyHints protected by lock It crashed kubelet by "concurrent map read and map write" --- pkg/kubelet/cm/topologymanager/scope.go | 18 +++++++++++++++++- .../cm/topologymanager/scope_container.go | 8 ++------ pkg/kubelet/cm/topologymanager/scope_pod.go | 7 +------ 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pkg/kubelet/cm/topologymanager/scope.go b/pkg/kubelet/cm/topologymanager/scope.go index d26636298b5..a7948c0b00c 100644 --- a/pkg/kubelet/cm/topologymanager/scope.go +++ b/pkg/kubelet/cm/topologymanager/scope.go @@ -68,10 +68,26 @@ func (s *scope) Name() string { return s.name } -func (s *scope) GetAffinity(podUID string, containerName string) TopologyHint { +func (s *scope) getTopologyHints(podUID string, containerName string) TopologyHint { + s.mutex.Lock() + defer s.mutex.Unlock() return s.podTopologyHints[podUID][containerName] } +func (s *scope) setTopologyHints(podUID string, containerName string, th TopologyHint) { + s.mutex.Lock() + defer s.mutex.Unlock() + + if s.podTopologyHints[podUID] == nil { + s.podTopologyHints[podUID] = make(map[string]TopologyHint) + } + s.podTopologyHints[podUID][containerName] = th +} + +func (s *scope) GetAffinity(podUID string, containerName string) TopologyHint { + return s.getTopologyHints(podUID, containerName) +} + func (s *scope) AddHintProvider(h HintProvider) { s.hintProviders = append(s.hintProviders, h) } diff --git a/pkg/kubelet/cm/topologymanager/scope_container.go b/pkg/kubelet/cm/topologymanager/scope_container.go index 4908e5f047e..0a221b0259b 100644 --- a/pkg/kubelet/cm/topologymanager/scope_container.go +++ b/pkg/kubelet/cm/topologymanager/scope_container.go @@ -55,13 +55,9 @@ func (s *containerScope) Admit(pod *v1.Pod) lifecycle.PodAdmitResult { if !admit { return topologyAffinityError() } - - if (s.podTopologyHints)[string(pod.UID)] == nil { - (s.podTopologyHints)[string(pod.UID)] = make(map[string]TopologyHint) - } - klog.Infof("[topologymanager] Topology Affinity for (pod: %v container: %v): %v", format.Pod(pod), container.Name, bestHint) - (s.podTopologyHints)[string(pod.UID)][container.Name] = bestHint + s.setTopologyHints(string(pod.UID), container.Name, bestHint) + err := s.allocateAlignedResources(pod, &container) if err != nil { return unexpectedAdmissionError(err) diff --git a/pkg/kubelet/cm/topologymanager/scope_pod.go b/pkg/kubelet/cm/topologymanager/scope_pod.go index 033577d2d64..11e66ac47c0 100644 --- a/pkg/kubelet/cm/topologymanager/scope_pod.go +++ b/pkg/kubelet/cm/topologymanager/scope_pod.go @@ -56,12 +56,7 @@ func (s *podScope) Admit(pod *v1.Pod) lifecycle.PodAdmitResult { for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) { klog.Infof("[topologymanager] Topology Affinity for (pod: %v container: %v): %v", format.Pod(pod), container.Name, bestHint) - - if (s.podTopologyHints)[string(pod.UID)] == nil { - (s.podTopologyHints)[string(pod.UID)] = make(map[string]TopologyHint) - } - - (s.podTopologyHints)[string(pod.UID)][container.Name] = bestHint + s.setTopologyHints(string(pod.UID), container.Name, bestHint) err := s.allocateAlignedResources(pod, &container) if err != nil {