From 93a237abd8255739dce59c84b6e1bccdde17b5d8 Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Wed, 28 Jul 2021 19:38:33 +0300 Subject: [PATCH] memory manager: do not clean admitted pods from the state Signed-off-by: Artyom Lukianov --- .../cm/memorymanager/memory_manager.go | 31 ++++- .../cm/memorymanager/memory_manager_test.go | 128 +++++++++++++++++- 2 files changed, 154 insertions(+), 5 deletions(-) diff --git a/pkg/kubelet/cm/memorymanager/memory_manager.go b/pkg/kubelet/cm/memorymanager/memory_manager.go index 4c8a786c5e8..3bf4dc0bf0e 100644 --- a/pkg/kubelet/cm/memorymanager/memory_manager.go +++ b/pkg/kubelet/cm/memorymanager/memory_manager.go @@ -124,6 +124,9 @@ type manager struct { // allocatableMemory holds the allocatable memory for each NUMA node allocatableMemory []state.Block + + // pendingAdmissionPod contain the pod during the admission phase + pendingAdmissionPod *v1.Pod } var _ Manager = &manager{} @@ -230,6 +233,10 @@ func (m *manager) GetMemoryNUMANodes(pod *v1.Pod, container *v1.Container) sets. // Allocate is called to pre-allocate memory resources during Pod admission. func (m *manager) Allocate(pod *v1.Pod, container *v1.Container) error { + // The pod is during the admission phase. We need to save the pod to avoid it + // being cleaned before the admission ended + m.setPodPendingAdmission(pod) + // Garbage collect any stranded resources before allocation m.removeStaleState() @@ -268,6 +275,10 @@ func (m *manager) State() state.Reader { // GetPodTopologyHints returns the topology hints for the topology manager func (m *manager) GetPodTopologyHints(pod *v1.Pod) map[string][]topologymanager.TopologyHint { + // The pod is during the admission phase. We need to save the pod to avoid it + // being cleaned before the admission ended + m.setPodPendingAdmission(pod) + // Garbage collect any stranded resources before providing TopologyHints m.removeStaleState() // Delegate to active policy @@ -276,6 +287,10 @@ func (m *manager) GetPodTopologyHints(pod *v1.Pod) map[string][]topologymanager. // GetTopologyHints returns the topology hints for the topology manager func (m *manager) GetTopologyHints(pod *v1.Pod, container *v1.Container) map[string][]topologymanager.TopologyHint { + // The pod is during the admission phase. We need to save the pod to avoid it + // being cleaned before the admission ended + m.setPodPendingAdmission(pod) + // Garbage collect any stranded resources before providing TopologyHints m.removeStaleState() // Delegate to active policy @@ -298,12 +313,15 @@ func (m *manager) removeStaleState() { m.Lock() defer m.Unlock() - // Get the list of active pods. - activePods := m.activePods() + // Get the list of admitted and active pods. + activeAndAdmittedPods := m.activePods() + if m.pendingAdmissionPod != nil { + activeAndAdmittedPods = append(activeAndAdmittedPods, m.pendingAdmissionPod) + } // Build a list of (podUID, containerName) pairs for all containers in all active Pods. activeContainers := make(map[string]map[string]struct{}) - for _, pod := range activePods { + for _, pod := range activeAndAdmittedPods { activeContainers[string(pod.UID)] = make(map[string]struct{}) for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) { activeContainers[string(pod.UID)][container.Name] = struct{}{} @@ -430,3 +448,10 @@ func (m *manager) GetAllocatableMemory() []state.Block { func (m *manager) GetMemory(podUID, containerName string) []state.Block { return m.state.GetMemoryBlocks(podUID, containerName) } + +func (m *manager) setPodPendingAdmission(pod *v1.Pod) { + m.Lock() + defer m.Unlock() + + m.pendingAdmissionPod = pod +} diff --git a/pkg/kubelet/cm/memorymanager/memory_manager_test.go b/pkg/kubelet/cm/memorymanager/memory_manager_test.go index d21615919b8..846b63c51f0 100644 --- a/pkg/kubelet/cm/memorymanager/memory_manager_test.go +++ b/pkg/kubelet/cm/memorymanager/memory_manager_test.go @@ -2019,6 +2019,129 @@ func TestNewManager(t *testing.T) { func TestGetTopologyHints(t *testing.T) { testCases := []testMemoryManager{ + { + description: "Successful hint generation", + policyName: policyTypeStatic, + machineInfo: returnMachineInfo(), + reserved: systemReservedMemory{ + 0: map[v1.ResourceName]uint64{ + v1.ResourceMemory: 1 * gb, + }, + 1: map[v1.ResourceName]uint64{ + v1.ResourceMemory: 1 * gb, + }, + }, + assignments: state.ContainerMemoryAssignments{ + "fakePod1": map[string][]state.Block{ + "fakeContainer1": { + { + NUMAAffinity: []int{0}, + Type: v1.ResourceMemory, + Size: 1 * gb, + }, + { + NUMAAffinity: []int{0}, + Type: hugepages1Gi, + Size: 1 * gb, + }, + }, + "fakeContainer2": { + { + NUMAAffinity: []int{0}, + Type: v1.ResourceMemory, + Size: 1 * gb, + }, + { + NUMAAffinity: []int{0}, + Type: hugepages1Gi, + Size: 1 * gb, + }, + }, + }, + }, + machineState: state.NUMANodeMap{ + 0: &state.NUMANodeState{ + Cells: []int{0}, + NumberOfAssignments: 4, + MemoryMap: map[v1.ResourceName]*state.MemoryTable{ + v1.ResourceMemory: { + Allocatable: 9 * gb, + Free: 7 * gb, + Reserved: 2 * gb, + SystemReserved: 1 * gb, + TotalMemSize: 10 * gb, + }, + hugepages1Gi: { + Allocatable: 5 * gb, + Free: 3 * gb, + Reserved: 2 * gb, + SystemReserved: 0 * gb, + TotalMemSize: 5 * gb, + }, + }, + }, + 1: &state.NUMANodeState{ + Cells: []int{1}, + NumberOfAssignments: 0, + MemoryMap: map[v1.ResourceName]*state.MemoryTable{ + v1.ResourceMemory: { + Allocatable: 9 * gb, + Free: 9 * gb, + Reserved: 0 * gb, + SystemReserved: 1 * gb, + TotalMemSize: 10 * gb, + }, + hugepages1Gi: { + Allocatable: 5 * gb, + Free: 5 * gb, + Reserved: 0, + SystemReserved: 0, + TotalMemSize: 5 * gb, + }, + }, + }, + }, + expectedError: nil, + expectedHints: map[string][]topologymanager.TopologyHint{ + string(v1.ResourceMemory): { + { + NUMANodeAffinity: newNUMAAffinity(0), + Preferred: true, + }, + { + NUMANodeAffinity: newNUMAAffinity(1), + Preferred: true, + }, + }, + string(hugepages1Gi): { + { + NUMANodeAffinity: newNUMAAffinity(0), + Preferred: true, + }, + { + NUMANodeAffinity: newNUMAAffinity(1), + Preferred: true, + }, + }, + }, + activePods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + UID: "fakePod1", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fakeContainer1", + }, + { + Name: "fakeContainer2", + }, + }, + }, + }, + }, + }, { description: "Successful hint generation", policyName: policyTypeStatic, @@ -2132,6 +2255,7 @@ func TestGetTopologyHints(t *testing.T) { }, }, }, + activePods: []*v1.Pod{}, }, } @@ -2144,14 +2268,14 @@ func TestGetTopologyHints(t *testing.T) { containerRuntime: mockRuntimeService{ err: nil, }, - activePods: func() []*v1.Pod { return nil }, + activePods: func() []*v1.Pod { return testCase.activePods }, podStatusProvider: mockPodStatusProvider{}, } mgr.sourcesReady = &sourcesReadyStub{} mgr.state.SetMachineState(testCase.machineState.Clone()) mgr.state.SetMemoryAssignments(testCase.assignments.Clone()) - pod := getPod("fakePod1", "fakeContainer1", requirementsGuaranteed) + pod := getPod("fakePod2", "fakeContainer1", requirementsGuaranteed) container := &pod.Spec.Containers[0] hints := mgr.GetTopologyHints(pod, container) if !reflect.DeepEqual(hints, testCase.expectedHints) {