From ba06be98e5ea783313cb804ebc4c458009d5210d Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Wed, 10 Nov 2021 14:31:48 +0200 Subject: [PATCH] e2e_node: provide an option to specify hugepages on the specific NUMA node On the multi NUMA node environment, kernel splits hugepages allocated under /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages file equally between NUMA nodes. That makes it harder to predict where several pods will start because the number of hugepages on each NUMA node will depend on the amount of NUMA nodes under the environment. The memory manager test will allocate hugepages on the specific NUMA node to make the test more predictable on the multi NUMA nodes environment. Signed-off-by: Artyom Lukianov --- test/e2e_node/hugepages_test.go | 20 +++++++++++++++----- test/e2e_node/memory_manager_test.go | 5 +++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/test/e2e_node/hugepages_test.go b/test/e2e_node/hugepages_test.go index cfd3d4188cb..773bbc3d308 100644 --- a/test/e2e_node/hugepages_test.go +++ b/test/e2e_node/hugepages_test.go @@ -118,7 +118,7 @@ func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity, } // configureHugePages attempts to allocate hugepages of the specified size -func configureHugePages(hugepagesSize int, hugepagesCount int) error { +func configureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error { // Compact memory to make bigger contiguous blocks of memory available // before allocating huge pages. // https://www.kernel.org/doc/Documentation/sysctl/vm.txt @@ -128,16 +128,26 @@ func configureHugePages(hugepagesSize int, hugepagesCount int) error { } } + // e.g. hugepages/hugepages-2048kB/nr_hugepages + hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile) + + // e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix) + if numaNodeID != nil { + // e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages + hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix) + } + // Reserve number of hugepages - // e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages" - command := fmt.Sprintf("echo %d > %s-%dkB/%s", hugepagesCount, hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile) + // e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" + command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile) if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { return err } // verify that the number of hugepages was updated // e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages" - command = fmt.Sprintf("cat %s-%dkB/%s", hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile) + command = fmt.Sprintf("cat %s", hugepagesFile) outData, err := exec.Command("/bin/sh", "-c", command).Output() if err != nil { return err @@ -258,7 +268,7 @@ var _ = SIGDescribe("HugePages [Serial] [Feature:HugePages][NodeSpecialFeature:H ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size)) gomega.Eventually(func() error { - if err := configureHugePages(size, count); err != nil { + if err := configureHugePages(size, count, nil); err != nil { return err } return nil diff --git a/test/e2e_node/memory_manager_test.go b/test/e2e_node/memory_manager_test.go index af3d948e025..bba3b5dc602 100644 --- a/test/e2e_node/memory_manager_test.go +++ b/test/e2e_node/memory_manager_test.go @@ -324,7 +324,7 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager if *is2MiHugepagesSupported { ginkgo.By("Configuring hugepages") gomega.Eventually(func() error { - return configureHugePages(hugepagesSize2M, hugepages2MiCount) + return configureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0)) }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) } }) @@ -356,7 +356,8 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager if *is2MiHugepagesSupported { ginkgo.By("Releasing allocated hugepages") gomega.Eventually(func() error { - return configureHugePages(hugepagesSize2M, 0) + // configure hugepages on the NUMA node 0 to avoid hugepages split across NUMA nodes + return configureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0)) }, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages") } })