github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 10:51:29 +00:00
Code Issues Projects Releases Wiki Activity

e2e_node: provide an option to specify hugepages on the specific NUMA node

Browse Source 
On the multi NUMA node environment, kernel splits hugepages allocated under
/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages file equally between NUMA nodes.
That makes it harder to predict where several pods will start because the number
of hugepages on each NUMA node will depend on the amount of NUMA nodes under the environment.
The memory manager test will allocate hugepages on the specific NUMA node to make
the test more predictable on the multi NUMA nodes environment.

Signed-off-by: Artyom Lukianov <alukiano@redhat.com>
This commit is contained in:
Artyom Lukianov 2021-11-10 14:31:48 +02:00
parent ea2011d72a
commit ba06be98e5
2 changed files with 18 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
test/e2e_node/hugepages_test.go
View File

@ -118,7 +118,7 @@ func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity,
}
// configureHugePages attempts to allocate hugepages of the specified size
func configureHugePages(hugepagesSize int, hugepagesCount int) error {
func configureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error {
// Compact memory to make bigger contiguous blocks of memory available
// before allocating huge pages.
// https://www.kernel.org/doc/Documentation/sysctl/vm.txt
@ -128,16 +128,26 @@ func configureHugePages(hugepagesSize int, hugepagesCount int) error {
}
}
// e.g. hugepages/hugepages-2048kB/nr_hugepages
hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile)
// e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix)
if numaNodeID != nil {
// e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix)
}
// Reserve number of hugepages
// e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages"
command := fmt.Sprintf("echo %d > %s-%dkB/%s", hugepagesCount, hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile)
// e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile)
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
return err
}
// verify that the number of hugepages was updated
// e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages"
command = fmt.Sprintf("cat %s-%dkB/%s", hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile)
command = fmt.Sprintf("cat %s", hugepagesFile)
outData, err := exec.Command("/bin/sh", "-c", command).Output()
if err != nil {
return err
@ -258,7 +268,7 @@ var _ = SIGDescribe("HugePages [Serial] [Feature:HugePages][NodeSpecialFeature:H
ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size))
gomega.Eventually(func() error {
if err := configureHugePages(size, count); err != nil {
if err := configureHugePages(size, count, nil); err != nil {
return err
}
return nil

5
test/e2e_node/memory_manager_test.go
View File

@ -324,7 +324,7 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
if *is2MiHugepagesSupported {
ginkgo.By("Configuring hugepages")
gomega.Eventually(func() error {
return configureHugePages(hugepagesSize2M, hugepages2MiCount)
return configureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0))
}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
}
})
@ -356,7 +356,8 @@ var _ = SIGDescribe("Memory Manager [Disruptive] [Serial] [Feature:MemoryManager
if *is2MiHugepagesSupported {
ginkgo.By("Releasing allocated hugepages")
gomega.Eventually(func() error {
return configureHugePages(hugepagesSize2M, 0)
// configure hugepages on the NUMA node 0 to avoid hugepages split across NUMA nodes
return configureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0))
}, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages")
}
})