From e8ea461bfdafd09747bf6e1ed37743f261f28968 Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Wed, 18 Nov 2020 00:22:43 +0200 Subject: [PATCH] memory manager: update all relevant part of code to use []MemoryReservation Signed-off-by: Artyom Lukianov --- pkg/features/kube_features.go | 2 +- pkg/kubelet/cm/memorymanager/BUILD | 4 +- .../cm/memorymanager/memory_manager.go | 49 +++-- .../cm/memorymanager/memory_manager_test.go | 202 +++++++++++++----- pkg/kubelet/types/BUILD | 1 - pkg/kubelet/types/types.go | 4 - test/e2e_node/memory_manager_test.go | 22 +- 7 files changed, 186 insertions(+), 98 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 0801411b39c..b22bc6659c4 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -126,7 +126,7 @@ const ( // owner: @cynepco3hahue(alukiano) @cezaryzukowski @k-wiatrzyk // alpha:: v1.20 - // Allows to set containers memory affinity according to NUMA topology + // Allows setting memory affinity for a container based on NUMA topology MemoryManager featuregate.Feature = "MemoryManager" // owner: @sjenning diff --git a/pkg/kubelet/cm/memorymanager/BUILD b/pkg/kubelet/cm/memorymanager/BUILD index 4d0552e2a82..2fd8286057c 100644 --- a/pkg/kubelet/cm/memorymanager/BUILD +++ b/pkg/kubelet/cm/memorymanager/BUILD @@ -14,13 +14,13 @@ go_library( deps = [ "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/core/v1/helper/qos:go_default_library", + "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/cm/containermap:go_default_library", "//pkg/kubelet/cm/memorymanager/state:go_default_library", "//pkg/kubelet/cm/topologymanager:go_default_library", "//pkg/kubelet/cm/topologymanager/bitmask:go_default_library", "//pkg/kubelet/config:go_default_library", "//pkg/kubelet/status:go_default_library", - "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library", @@ -37,12 +37,12 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/cm/containermap:go_default_library", "//pkg/kubelet/cm/memorymanager/state:go_default_library", "//pkg/kubelet/cm/topologymanager:go_default_library", "//pkg/kubelet/cm/topologymanager/bitmask:go_default_library", "//pkg/kubelet/lifecycle:go_default_library", - "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/format:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/pkg/kubelet/cm/memorymanager/memory_manager.go b/pkg/kubelet/cm/memorymanager/memory_manager.go index 4929eaa2699..7840d6d0b25 100644 --- a/pkg/kubelet/cm/memorymanager/memory_manager.go +++ b/pkg/kubelet/cm/memorymanager/memory_manager.go @@ -29,12 +29,12 @@ import ( runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/klog/v2" corev1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/memorymanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/status" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) // memoryManagerStateFileName is the file name where memory manager stores its state @@ -67,7 +67,7 @@ type Manager interface { Allocate(pod *v1.Pod, container *v1.Container) error // RemoveContainer is called after Kubelet decides to kill or delete a - // container. After this call, any memory allocated to the container are freed. + // container. After this call, any memory allocated to the container is freed. RemoveContainer(containerID string) error // State returns a read-only interface to the internal memory manager state. @@ -119,7 +119,7 @@ type manager struct { var _ Manager = &manager{} // NewManager returns new instance of the memory manager -func NewManager(policyName string, machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, reservedMemory kubetypes.NUMANodeResources, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) { +func NewManager(policyName string, machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, reservedMemory []kubeletconfig.MemoryReservation, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) { var policy Policy switch policyType(policyName) { @@ -321,7 +321,7 @@ func (m *manager) policyRemoveContainerByRef(podUID string, containerName string return err } -func getTotalMemoryTypeReserved(machineInfo *cadvisorapi.MachineInfo, reservedMemory kubetypes.NUMANodeResources) map[v1.ResourceName]resource.Quantity { +func getTotalMemoryTypeReserved(machineInfo *cadvisorapi.MachineInfo, reservedMemory []kubeletconfig.MemoryReservation) map[v1.ResourceName]resource.Quantity { totalMemoryType := map[v1.ResourceName]resource.Quantity{} numaNodes := map[int]bool{} @@ -329,24 +329,24 @@ func getTotalMemoryTypeReserved(machineInfo *cadvisorapi.MachineInfo, reservedMe numaNodes[numaNode.Id] = true } - for nodeID, node := range reservedMemory { - if !numaNodes[nodeID] { - klog.Warningf("The NUMA node %d specified under --reserved- memory does not exist on the machine", nodeID) + for _, reservation := range reservedMemory { + if !numaNodes[int(reservation.NumaNode)] { + klog.Warningf("The NUMA node %d specified under --reserved-memory does not exist on the machine", reservation.NumaNode) continue } - for memType, memVal := range node { - if totalMem, exists := totalMemoryType[memType]; exists { - memVal.Add(totalMem) + for resourceName, q := range reservation.Limits { + if value, ok := totalMemoryType[resourceName]; ok { + q.Add(value) } - totalMemoryType[memType] = memVal + totalMemoryType[resourceName] = q } } return totalMemoryType } -func validateReservedMemory(machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, reservedMemory kubetypes.NUMANodeResources) error { +func validateReservedMemory(machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, reservedMemory []kubeletconfig.MemoryReservation) error { totalMemoryType := getTotalMemoryTypeReserved(machineInfo, reservedMemory) commonMemoryTypeSet := make(map[v1.ResourceName]bool) @@ -382,32 +382,31 @@ func validateReservedMemory(machineInfo *cadvisorapi.MachineInfo, nodeAllocatabl return nil } -func convertReserved(machineInfo *cadvisorapi.MachineInfo, reservedMemory kubetypes.NUMANodeResources) (systemReservedMemory, error) { - preReservedMemoryConverted := make(map[int]map[v1.ResourceName]uint64) +func convertReserved(machineInfo *cadvisorapi.MachineInfo, reservedMemory []kubeletconfig.MemoryReservation) (systemReservedMemory, error) { + reservedMemoryConverted := make(map[int]map[v1.ResourceName]uint64) for _, node := range machineInfo.Topology { - preReservedMemoryConverted[node.Id] = make(map[v1.ResourceName]uint64) + reservedMemoryConverted[node.Id] = make(map[v1.ResourceName]uint64) } - for numaIndex := range reservedMemory { - for memoryType := range reservedMemory[numaIndex] { - tmp := reservedMemory[numaIndex][memoryType] - if val, success := tmp.AsInt64(); success { - preReservedMemoryConverted[numaIndex][memoryType] = uint64(val) - } else { + for _, reservation := range reservedMemory { + for resourceName, q := range reservation.Limits { + val, success := q.AsInt64() + if !success { return nil, fmt.Errorf("could not covert a variable of type Quantity to int64") } + reservedMemoryConverted[int(reservation.NumaNode)][resourceName] = uint64(val) } } - return preReservedMemoryConverted, nil + return reservedMemoryConverted, nil } -func getSystemReservedMemory(machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, preReservedMemory kubetypes.NUMANodeResources) (systemReservedMemory, error) { - if err := validateReservedMemory(machineInfo, nodeAllocatableReservation, preReservedMemory); err != nil { +func getSystemReservedMemory(machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, reservedMemory []kubeletconfig.MemoryReservation) (systemReservedMemory, error) { + if err := validateReservedMemory(machineInfo, nodeAllocatableReservation, reservedMemory); err != nil { return nil, err } - reservedMemoryConverted, err := convertReserved(machineInfo, preReservedMemory) + reservedMemoryConverted, err := convertReserved(machineInfo, reservedMemory) if err != nil { return nil, err } diff --git a/pkg/kubelet/cm/memorymanager/memory_manager_test.go b/pkg/kubelet/cm/memorymanager/memory_manager_test.go index e336b929719..e1d0f20d86d 100644 --- a/pkg/kubelet/cm/memorymanager/memory_manager_test.go +++ b/pkg/kubelet/cm/memorymanager/memory_manager_test.go @@ -24,6 +24,8 @@ import ( "strings" "testing" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + cadvisorapi "github.com/google/cadvisor/info/v1" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -34,7 +36,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/memorymanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) const ( @@ -59,7 +60,7 @@ type testMemoryManager struct { nodeAllocatableReservation v1.ResourceList policyName policyType affinity topologymanager.Store - systemReservedMemory kubetypes.NUMANodeResources + systemReservedMemory []kubeletconfig.MemoryReservation expectedHints map[string][]topologymanager.TopologyHint expectedReserved systemReservedMemory reserved systemReservedMemory @@ -69,7 +70,7 @@ type testMemoryManager struct { } func returnPolicyByName(testCase testMemoryManager) Policy { - switch policyType(testCase.policyName) { + switch testCase.policyName { case policyTypeMock: return &mockPolicy{ err: fmt.Errorf("fake reg error"), @@ -83,8 +84,6 @@ func returnPolicyByName(testCase testMemoryManager) Policy { return nil } -type nodeResources map[v1.ResourceName]resource.Quantity - type mockPolicy struct { err error } @@ -158,22 +157,27 @@ func TestValidateReservedMemory(t *testing.T) { description string nodeAllocatableReservation v1.ResourceList machineInfo *cadvisorapi.MachineInfo - systemReservedMemory kubetypes.NUMANodeResources + systemReservedMemory []kubeletconfig.MemoryReservation expectedError string }{ { "Node Allocatable not set, reserved not set", v1.ResourceList{}, machineInfo, - kubetypes.NUMANodeResources{}, + []kubeletconfig.MemoryReservation{}, "", }, { "Node Allocatable set to zero, reserved set to zero", v1.ResourceList{v1.ResourceMemory: *resource.NewQuantity(0, resource.DecimalSI)}, machineInfo, - kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(0, resource.DecimalSI)}, + []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(0, resource.DecimalSI), + }, + }, }, "", }, @@ -181,8 +185,13 @@ func TestValidateReservedMemory(t *testing.T) { "Node Allocatable not set (equal zero), reserved set", v1.ResourceList{}, machineInfo, - kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI)}, + []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), + }, + }, }, fmt.Sprintf(msgNotEqual, v1.ResourceMemory), }, @@ -190,15 +199,20 @@ func TestValidateReservedMemory(t *testing.T) { "Node Allocatable set, reserved not set", v1.ResourceList{hugepages2M: *resource.NewQuantity(5, resource.DecimalSI)}, machineInfo, - kubetypes.NUMANodeResources{}, + []kubeletconfig.MemoryReservation{}, fmt.Sprintf(msgNotEqual, hugepages2M), }, { "Reserved not equal to Node Allocatable", v1.ResourceList{v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI)}, machineInfo, - kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI)}, + []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), + }, + }, }, fmt.Sprintf(msgNotEqual, v1.ResourceMemory), }, @@ -206,9 +220,19 @@ func TestValidateReservedMemory(t *testing.T) { "Reserved contains the NUMA node that does not exist under the machine", v1.ResourceList{v1.ResourceMemory: *resource.NewQuantity(17, resource.DecimalSI)}, machineInfo, - kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI)}, - 2: nodeResources{v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI)}, + []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), + }, + }, + { + NumaNode: 2, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI), + }, + }, }, fmt.Sprintf(msgNotEqual, v1.ResourceMemory), }, @@ -218,12 +242,22 @@ func TestValidateReservedMemory(t *testing.T) { hugepages2M: *resource.NewQuantity(77, resource.DecimalSI), hugepages1G: *resource.NewQuantity(13, resource.DecimalSI)}, machineInfo, - kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), - hugepages2M: *resource.NewQuantity(70, resource.DecimalSI), - hugepages1G: *resource.NewQuantity(13, resource.DecimalSI)}, - 1: nodeResources{v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI), - hugepages2M: *resource.NewQuantity(7, resource.DecimalSI)}, + []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), + hugepages2M: *resource.NewQuantity(70, resource.DecimalSI), + hugepages1G: *resource.NewQuantity(13, resource.DecimalSI), + }, + }, + { + NumaNode: 1, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI), + hugepages2M: *resource.NewQuantity(7, resource.DecimalSI), + }, + }, }, "", }, @@ -233,12 +267,22 @@ func TestValidateReservedMemory(t *testing.T) { hugepages2M: *resource.NewQuantity(14, resource.DecimalSI), hugepages1G: *resource.NewQuantity(13, resource.DecimalSI)}, machineInfo, - kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), - hugepages2M: *resource.NewQuantity(70, resource.DecimalSI), - hugepages1G: *resource.NewQuantity(13, resource.DecimalSI)}, - 1: nodeResources{v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI), - hugepages2M: *resource.NewQuantity(7, resource.DecimalSI)}, + []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), + hugepages2M: *resource.NewQuantity(70, resource.DecimalSI), + hugepages1G: *resource.NewQuantity(13, resource.DecimalSI), + }, + }, + { + NumaNode: 1, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI), + hugepages2M: *resource.NewQuantity(7, resource.DecimalSI), + }, + }, }, fmt.Sprintf(msgNotEqual, hugepages2M), @@ -266,13 +310,13 @@ func TestConvertPreReserved(t *testing.T) { testCases := []struct { description string - systemReserved kubetypes.NUMANodeResources + systemReserved []kubeletconfig.MemoryReservation systemReservedExpected systemReservedMemory expectedError string }{ { "Empty", - kubetypes.NUMANodeResources{}, + []kubeletconfig.MemoryReservation{}, systemReservedMemory{ 0: map[v1.ResourceName]uint64{}, 1: map[v1.ResourceName]uint64{}, @@ -281,10 +325,15 @@ func TestConvertPreReserved(t *testing.T) { }, { "Single NUMA node is reserved", - kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), - hugepages2M: *resource.NewQuantity(70, resource.DecimalSI), - hugepages1G: *resource.NewQuantity(13, resource.DecimalSI)}, + []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), + hugepages2M: *resource.NewQuantity(70, resource.DecimalSI), + hugepages1G: *resource.NewQuantity(13, resource.DecimalSI), + }, + }, }, systemReservedMemory{ 0: map[v1.ResourceName]uint64{ @@ -298,12 +347,22 @@ func TestConvertPreReserved(t *testing.T) { }, { "Both NUMA nodes are reserved", - kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), - hugepages2M: *resource.NewQuantity(70, resource.DecimalSI), - hugepages1G: *resource.NewQuantity(13, resource.DecimalSI)}, - 1: nodeResources{v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI), - hugepages2M: *resource.NewQuantity(7, resource.DecimalSI)}, + []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(12, resource.DecimalSI), + hugepages2M: *resource.NewQuantity(70, resource.DecimalSI), + hugepages1G: *resource.NewQuantity(13, resource.DecimalSI), + }, + }, + { + NumaNode: 1, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(5, resource.DecimalSI), + hugepages2M: *resource.NewQuantity(7, resource.DecimalSI), + }, + }, }, systemReservedMemory{ 0: map[v1.ResourceName]uint64{ @@ -336,7 +395,7 @@ func TestGetSystemReservedMemory(t *testing.T) { { description: "Should return empty map when reservation is not done", nodeAllocatableReservation: v1.ResourceList{}, - systemReservedMemory: kubetypes.NUMANodeResources{}, + systemReservedMemory: []kubeletconfig.MemoryReservation{}, expectedReserved: systemReservedMemory{ 0: {}, 1: {}, @@ -347,8 +406,13 @@ func TestGetSystemReservedMemory(t *testing.T) { { description: "Should return error when Allocatable reservation is not equal pre reserved memory", nodeAllocatableReservation: v1.ResourceList{}, - systemReservedMemory: kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI)}, + systemReservedMemory: []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI), + }, + }, }, expectedReserved: nil, expectedError: fmt.Errorf("the total amount of memory of type \"memory\" is not equal to the value determined by Node Allocatable feature"), @@ -357,9 +421,19 @@ func TestGetSystemReservedMemory(t *testing.T) { { description: "Reserved should be equal to systemReservedMemory", nodeAllocatableReservation: v1.ResourceList{v1.ResourceMemory: *resource.NewQuantity(2*gb, resource.BinarySI)}, - systemReservedMemory: kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI)}, - 1: nodeResources{v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI)}, + systemReservedMemory: []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI), + }, + }, + { + NumaNode: 1, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI), + }, + }, }, expectedReserved: systemReservedMemory{ 0: map[v1.ResourceName]uint64{ @@ -2063,9 +2137,15 @@ func TestNewManager(t *testing.T) { policyName: policyTypeStatic, machineInfo: machineInfo, nodeAllocatableReservation: v1.ResourceList{v1.ResourceMemory: *resource.NewQuantity(2*gb, resource.BinarySI)}, - systemReservedMemory: kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI)}, - 1: nodeResources{v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI)}, + systemReservedMemory: []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI)}, + }, + { + NumaNode: 1, + Limits: v1.ResourceList{v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI)}, + }, }, affinity: topologymanager.NewFakeManager(), expectedError: nil, @@ -2076,9 +2156,19 @@ func TestNewManager(t *testing.T) { policyName: policyTypeStatic, machineInfo: machineInfo, nodeAllocatableReservation: v1.ResourceList{v1.ResourceMemory: *resource.NewQuantity(2*gb, resource.BinarySI)}, - systemReservedMemory: kubetypes.NUMANodeResources{ - 0: nodeResources{v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI)}, - 1: nodeResources{v1.ResourceMemory: *resource.NewQuantity(2*gb, resource.BinarySI)}, + systemReservedMemory: []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(gb, resource.BinarySI), + }, + }, + { + NumaNode: 1, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(2*gb, resource.BinarySI), + }, + }, }, affinity: topologymanager.NewFakeManager(), expectedError: fmt.Errorf("the total amount of memory of type %q is not equal to the value determined by Node Allocatable feature", v1.ResourceMemory), @@ -2089,7 +2179,7 @@ func TestNewManager(t *testing.T) { policyName: policyTypeStatic, machineInfo: machineInfo, nodeAllocatableReservation: v1.ResourceList{}, - systemReservedMemory: kubetypes.NUMANodeResources{}, + systemReservedMemory: []kubeletconfig.MemoryReservation{}, affinity: topologymanager.NewFakeManager(), expectedError: fmt.Errorf("[memorymanager] you should specify the system reserved memory"), expectedReserved: expectedReserved, @@ -2099,7 +2189,7 @@ func TestNewManager(t *testing.T) { policyName: "fake", machineInfo: machineInfo, nodeAllocatableReservation: v1.ResourceList{}, - systemReservedMemory: kubetypes.NUMANodeResources{}, + systemReservedMemory: []kubeletconfig.MemoryReservation{}, affinity: topologymanager.NewFakeManager(), expectedError: fmt.Errorf("unknown policy: \"fake\""), expectedReserved: expectedReserved, @@ -2109,7 +2199,7 @@ func TestNewManager(t *testing.T) { policyName: policyTypeNone, machineInfo: machineInfo, nodeAllocatableReservation: v1.ResourceList{}, - systemReservedMemory: kubetypes.NUMANodeResources{}, + systemReservedMemory: []kubeletconfig.MemoryReservation{}, affinity: topologymanager.NewFakeManager(), expectedError: nil, expectedReserved: expectedReserved, diff --git a/pkg/kubelet/types/BUILD b/pkg/kubelet/types/BUILD index 06357196f2b..2f4e1abe191 100644 --- a/pkg/kubelet/types/BUILD +++ b/pkg/kubelet/types/BUILD @@ -20,7 +20,6 @@ go_library( deps = [ "//pkg/apis/scheduling:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", ], diff --git a/pkg/kubelet/types/types.go b/pkg/kubelet/types/types.go index 8dc13bcc791..2ed7f1c991e 100644 --- a/pkg/kubelet/types/types.go +++ b/pkg/kubelet/types/types.go @@ -21,7 +21,6 @@ import ( "time" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" ) @@ -114,6 +113,3 @@ type ResolvedPodUID types.UID // MirrorPodUID is a pod UID for a mirror pod. type MirrorPodUID types.UID - -// NUMANodeResources is a set of (resource name, quantity) pairs for each NUMA node. -type NUMANodeResources map[int]map[v1.ResourceName]resource.Quantity diff --git a/test/e2e_node/memory_manager_test.go b/test/e2e_node/memory_manager_test.go index fb02b8aed46..d25565f80a8 100644 --- a/test/e2e_node/memory_manager_test.go +++ b/test/e2e_node/memory_manager_test.go @@ -46,9 +46,6 @@ import ( const ( evictionHardMemory = "memory.available" memoryManagerStateFile = "/var/lib/kubelet/memory_manager_state" - reservedLimit = "limit" - reservedNUMANode = "numa-node" - reservedType = "type" resourceMemory = "memory" staticPolicy = "static" nonePolicy = "none" @@ -159,7 +156,7 @@ func getMemoryManagerState() (*state.MemoryManagerCheckpoint, error) { type kubeletParams struct { memoryManagerFeatureGate bool memoryManagerPolicy string - systemReservedMemory []map[string]string + systemReservedMemory []kubeletconfig.MemoryReservation systemReserved map[string]string kubeReserved map[string]string evictionHard map[string]string @@ -200,10 +197,10 @@ func getUpdatedKubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration, params // update reserved memory if newCfg.ReservedMemory == nil { - newCfg.ReservedMemory = []map[string]string{} + newCfg.ReservedMemory = []kubeletconfig.MemoryReservation{} } - for _, p := range params.systemReservedMemory { - newCfg.ReservedMemory = append(newCfg.ReservedMemory, p) + for _, memoryReservation := range params.systemReservedMemory { + newCfg.ReservedMemory = append(newCfg.ReservedMemory, memoryReservation) } return newCfg @@ -259,10 +256,17 @@ var _ = SIGDescribe("Memory Manager [Serial] [Feature:MemoryManager][NodeAlphaFe ) f := framework.NewDefaultFramework("memory-manager-test") + + memoryQuantatity := resource.MustParse("1100Mi") defaultKubeParams := &kubeletParams{ memoryManagerFeatureGate: true, - systemReservedMemory: []map[string]string{ - {reservedNUMANode: "0", reservedType: resourceMemory, reservedLimit: "1100Mi"}, + systemReservedMemory: []kubeletconfig.MemoryReservation{ + { + NumaNode: 0, + Limits: v1.ResourceList{ + resourceMemory: memoryQuantatity, + }, + }, }, systemReserved: map[string]string{resourceMemory: "500Mi"}, kubeReserved: map[string]string{resourceMemory: "500Mi"},