From 681905706d43c91403bace725a600d35e02d59c1 Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Mon, 10 May 2021 15:28:08 +0300 Subject: [PATCH] e2e node: provide tests for memory manager pod resources metrics - verify memory manager data returned by `GetAllocatableResources` - verify pod container memory manager data Signed-off-by: Artyom Lukianov --- test/e2e_node/memory_manager_test.go | 174 +++++++++++++++++++++++++-- 1 file changed, 163 insertions(+), 11 deletions(-) diff --git a/test/e2e_node/memory_manager_test.go b/test/e2e_node/memory_manager_test.go index 4a063c66900..d098d10b356 100644 --- a/test/e2e_node/memory_manager_test.go +++ b/test/e2e_node/memory_manager_test.go @@ -31,9 +31,12 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/memorymanager/state" + "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -153,13 +156,33 @@ func getMemoryManagerState() (*state.MemoryManagerCheckpoint, error) { return memoryManagerCheckpoint, nil } +func getAllocatableMemoryFromStateFile(s *state.MemoryManagerCheckpoint) []state.Block { + var allocatableMemory []state.Block + for numaNodeID, numaNodeState := range s.MachineState { + for resourceName, memoryTable := range numaNodeState.MemoryMap { + if memoryTable.Allocatable == 0 { + continue + } + + block := state.Block{ + NUMAAffinity: []int{numaNodeID}, + Type: resourceName, + Size: memoryTable.Allocatable, + } + allocatableMemory = append(allocatableMemory, block) + } + } + return allocatableMemory +} + type kubeletParams struct { - memoryManagerFeatureGate bool - memoryManagerPolicy string - systemReservedMemory []kubeletconfig.MemoryReservation - systemReserved map[string]string - kubeReserved map[string]string - evictionHard map[string]string + memoryManagerFeatureGate bool + podResourcesGetAllocatableFeatureGate bool + memoryManagerPolicy string + systemReservedMemory []kubeletconfig.MemoryReservation + systemReserved map[string]string + kubeReserved map[string]string + evictionHard map[string]string } func getUpdatedKubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration, params *kubeletParams) *kubeletconfig.KubeletConfiguration { @@ -169,6 +192,8 @@ func getUpdatedKubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration, params newCfg.FeatureGates = map[string]bool{} } newCfg.FeatureGates["MemoryManager"] = params.memoryManagerFeatureGate + newCfg.FeatureGates["KubeletPodResourcesGetAllocatable"] = params.podResourcesGetAllocatableFeatureGate + newCfg.MemoryManagerPolicy = params.memoryManagerPolicy // update system-reserved @@ -257,14 +282,15 @@ var _ = SIGDescribe("Memory Manager [Serial] [Feature:MemoryManager][NodeAlphaFe f := framework.NewDefaultFramework("memory-manager-test") - memoryQuantatity := resource.MustParse("1100Mi") + memoryQuantity := resource.MustParse("1100Mi") defaultKubeParams := &kubeletParams{ - memoryManagerFeatureGate: true, + memoryManagerFeatureGate: true, + podResourcesGetAllocatableFeatureGate: true, systemReservedMemory: []kubeletconfig.MemoryReservation{ { NumaNode: 0, Limits: v1.ResourceList{ - resourceMemory: memoryQuantatity, + resourceMemory: memoryQuantity, }, }, }, @@ -367,12 +393,14 @@ var _ = SIGDescribe("Memory Manager [Serial] [Feature:MemoryManager][NodeAlphaFe } } - testPod = makeMemoryManagerPod(ctnParams[0].ctnName, initCtnParams, ctnParams) + if len(ctnParams) > 0 { + testPod = makeMemoryManagerPod(ctnParams[0].ctnName, initCtnParams, ctnParams) + } }) ginkgo.JustAfterEach(func() { // delete the test pod - if testPod.Name != "" { + if testPod != nil && testPod.Name != "" { f.PodClient().DeleteSync(testPod.Name, metav1.DeleteOptions{}, 2*time.Minute) } @@ -407,6 +435,48 @@ var _ = SIGDescribe("Memory Manager [Serial] [Feature:MemoryManager][NodeAlphaFe initCtnParams = []memoryManagerCtnAttributes{} }) + // TODO: move the test to pod resource API test suite, see - https://github.com/kubernetes/kubernetes/issues/101945 + ginkgo.It("should report memory data during request to pod resources GetAllocatableResources", func() { + endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + framework.ExpectNoError(err) + + cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) + framework.ExpectNoError(err) + defer conn.Close() + + resp, err := cli.GetAllocatableResources(context.TODO(), &kubeletpodresourcesv1.AllocatableResourcesRequest{}) + framework.ExpectNoError(err) + gomega.Expect(resp.Memory).ToNot(gomega.BeEmpty()) + + stateData, err := getMemoryManagerState() + framework.ExpectNoError(err) + + stateAllocatableMemory := getAllocatableMemoryFromStateFile(stateData) + framework.ExpectEqual(len(resp.Memory), len(stateAllocatableMemory)) + + for _, containerMemory := range resp.Memory { + gomega.Expect(containerMemory.Topology).NotTo(gomega.BeNil()) + framework.ExpectEqual(len(containerMemory.Topology.Nodes), 1) + gomega.Expect(containerMemory.Topology.Nodes[0]).NotTo(gomega.BeNil()) + + numaNodeID := int(containerMemory.Topology.Nodes[0].ID) + for _, numaStateMemory := range stateAllocatableMemory { + framework.ExpectEqual(len(numaStateMemory.NUMAAffinity), 1) + if numaNodeID != numaStateMemory.NUMAAffinity[0] { + continue + } + + if containerMemory.MemoryType != string(numaStateMemory.Type) { + continue + } + + gomega.Expect(containerMemory.Size_).To(gomega.BeEquivalentTo(numaStateMemory.Size)) + } + } + + gomega.Expect(resp.Memory).ToNot(gomega.BeEmpty()) + }) + ginkgo.When("guaranteed pod has init and app containers", func() { ginkgo.BeforeEach(func() { // override containers parameters @@ -499,6 +569,48 @@ var _ = SIGDescribe("Memory Manager [Serial] [Feature:MemoryManager][NodeAlphaFe verifyMemoryPinning(testPod2, []int{0}) }) + // TODO: move the test to pod resource API test suite, see - https://github.com/kubernetes/kubernetes/issues/101945 + ginkgo.It("should report memory data for each guaranteed pod and container during request to pod resources List", func() { + ginkgo.By("Running the test pod and the test pod 2") + testPod = f.PodClient().CreateSync(testPod) + + ginkgo.By("Running the test pod 2") + testPod2 = f.PodClient().CreateSync(testPod2) + + endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + framework.ExpectNoError(err) + + cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) + framework.ExpectNoError(err) + defer conn.Close() + + resp, err := cli.List(context.TODO(), &kubeletpodresourcesv1.ListPodResourcesRequest{}) + framework.ExpectNoError(err) + + for _, pod := range []*v1.Pod{testPod, testPod2} { + for _, podResource := range resp.PodResources { + if podResource.Name != pod.Name { + continue + } + + for _, c := range pod.Spec.Containers { + for _, containerResource := range podResource.Containers { + if containerResource.Name != c.Name { + continue + } + + for _, containerMemory := range containerResource.Memory { + q := c.Resources.Limits[v1.ResourceName(containerMemory.MemoryType)] + value, ok := q.AsInt64() + gomega.Expect(ok).To(gomega.BeTrue()) + gomega.Expect(value).To(gomega.BeEquivalentTo(containerMemory.Size_)) + } + } + } + } + } + }) + ginkgo.JustAfterEach(func() { // delete the test pod 2 if testPod2.Name != "" { @@ -604,6 +716,46 @@ var _ = SIGDescribe("Memory Manager [Serial] [Feature:MemoryManager][NodeAlphaFe } }) + // TODO: move the test to pod resource API test suite, see - https://github.com/kubernetes/kubernetes/issues/101945 + ginkgo.It("should not report any memory data during request to pod resources GetAllocatableResources", func() { + endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + framework.ExpectNoError(err) + + cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) + framework.ExpectNoError(err) + defer conn.Close() + + resp, err := cli.GetAllocatableResources(context.TODO(), &kubeletpodresourcesv1.AllocatableResourcesRequest{}) + framework.ExpectNoError(err) + + gomega.Expect(resp.Memory).To(gomega.BeEmpty()) + }) + + // TODO: move the test to pod resource API test suite, see - https://github.com/kubernetes/kubernetes/issues/101945 + ginkgo.It("should not report any memory data during request to pod resources List", func() { + testPod = f.PodClient().CreateSync(testPod) + + endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + framework.ExpectNoError(err) + + cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) + framework.ExpectNoError(err) + defer conn.Close() + + resp, err := cli.List(context.TODO(), &kubeletpodresourcesv1.ListPodResourcesRequest{}) + framework.ExpectNoError(err) + + for _, podResource := range resp.PodResources { + if podResource.Name != testPod.Name { + continue + } + + for _, containerResource := range podResource.Containers { + gomega.Expect(containerResource.Memory).To(gomega.BeEmpty()) + } + } + }) + ginkgo.It("should succeed to start the pod", func() { testPod = f.PodClient().CreateSync(testPod)