From a4b367a6a315db6ccf3c40a4a0374372b4307acf Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Sun, 7 Jun 2020 11:38:44 +0300 Subject: [PATCH] Refactor and add new tests to hugepages e2e tests Add tests to cover usage of multiple hugepages with different page sizes under the same pod. Signed-off-by: Artyom Lukianov --- test/e2e_node/hugepages_test.go | 458 +++++++++++++++++++++++--------- 1 file changed, 327 insertions(+), 131 deletions(-) diff --git a/test/e2e_node/hugepages_test.go b/test/e2e_node/hugepages_test.go index 145eb38c447..95a99574d1f 100644 --- a/test/e2e_node/hugepages_test.go +++ b/test/e2e_node/hugepages_test.go @@ -25,6 +25,9 @@ import ( "strings" "time" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,30 +37,53 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - imageutils "k8s.io/kubernetes/test/utils/image" +) - "github.com/onsi/ginkgo" - "github.com/onsi/gomega" +const ( + hugepagesSize2M = 2048 + hugepagesSize1G = 1048576 + hugepagesDirPrefix = "/sys/kernel/mm/hugepages/hugepages" + hugepagesCapacityFile = "nr_hugepages" + hugepagesResourceName2Mi = "hugepages-2Mi" + hugepagesResourceName1Gi = "hugepages-1Gi" + hugepagesCgroup2MB = "hugetlb.2MB" + hugepagesCgroup1GB = "hugetlb.1GB" + mediumHugepages = "HugePages" + mediumHugepages2Mi = "HugePages-2Mi" + mediumHugepages1Gi = "HugePages-1Gi" +) + +var ( + resourceToSize = map[string]int{ + hugepagesResourceName2Mi: hugepagesSize2M, + hugepagesResourceName1Gi: hugepagesSize1G, + } + resourceToCgroup = map[string]string{ + hugepagesResourceName2Mi: hugepagesCgroup2MB, + hugepagesResourceName1Gi: hugepagesCgroup1GB, + } ) // makePodToVerifyHugePages returns a pod that verifies specified cgroup with hugetlb -func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity) *v1.Pod { +func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity, hugepagesCgroup string) *v1.Pod { // convert the cgroup name to its literal form - cgroupFsName := "" cgroupName := cm.NewCgroupName(cm.RootCgroupName, defaultNodeAllocatableCgroup, baseName) + cgroupFsName := "" if framework.TestContext.KubeletConfig.CgroupDriver == "systemd" { cgroupFsName = cgroupName.ToSystemd() } else { cgroupFsName = cgroupName.ToCgroupfs() } - command := "" + hugetlbLimitFile := "" // this command takes the expected value and compares it against the actual value for the pod cgroup hugetlb.2MB. if IsCgroup2UnifiedMode() { - command = fmt.Sprintf("expected=%v; actual=$(cat /tmp/%v/hugetlb.2MB.max); if [ \"$expected\" -ne \"$actual\" ]; then exit 1; fi; ", hugePagesLimit.Value(), cgroupFsName) + hugetlbLimitFile = fmt.Sprintf("/tmp/%s/%s.max", cgroupFsName, hugepagesCgroup) } else { - command = fmt.Sprintf("expected=%v; actual=$(cat /tmp/hugetlb/%v/hugetlb.2MB.limit_in_bytes); if [ \"$expected\" -ne \"$actual\" ]; then exit 1; fi; ", hugePagesLimit.Value(), cgroupFsName) + hugetlbLimitFile = fmt.Sprintf("/tmp/hugetlb/%s/%s.limit_in_bytes", cgroupFsName, hugepagesCgroup) } + + command := fmt.Sprintf("expected=%v; actual=$(cat %v); if [ \"$expected\" -ne \"$actual\" ]; then exit 1; fi; ", hugePagesLimit.Value(), hugetlbLimitFile) framework.Logf("Pod to run command: %v", command) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -91,98 +117,105 @@ func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity) return pod } -// configureHugePages attempts to allocate 10Mi of 2Mi hugepages for testing purposes -func configureHugePages() error { +// configureHugePages attempts to allocate hugepages of the specified size +func configureHugePages(hugepagesSize int, hugepagesCount int) error { // Compact memory to make bigger contiguous blocks of memory available // before allocating huge pages. // https://www.kernel.org/doc/Documentation/sysctl/vm.txt if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil { - err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run() - if err != nil { + if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil { return err } } - err := exec.Command("/bin/sh", "-c", "echo 5 > /proc/sys/vm/nr_hugepages").Run() - if err != nil { - return err - } - outData, err := exec.Command("/bin/sh", "-c", "cat /proc/meminfo | grep 'HugePages_Total' | awk '{print $2}'").Output() + + // Reserve number of hugepages + // e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages" + command := fmt.Sprintf("echo %d > %s-%dkB/%s", hugepagesCount, hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile) + if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { + return err + } + + // verify that the number of hugepages was updated + // e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages" + command = fmt.Sprintf("cat %s-%dkB/%s", hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile) + outData, err := exec.Command("/bin/sh", "-c", command).Output() if err != nil { return err } + numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData))) if err != nil { return err } - framework.Logf("HugePages_Total is set to %v", numHugePages) - if numHugePages == 5 { + + framework.Logf("Hugepages total is set to %v", numHugePages) + if numHugePages == hugepagesCount { return nil } - return fmt.Errorf("expected hugepages %v, but found %v", 5, numHugePages) + + return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages) } -// releaseHugePages releases all pre-allocated hugepages -func releaseHugePages() error { - return exec.Command("/bin/sh", "-c", "echo 0 > /proc/sys/vm/nr_hugepages").Run() -} - -// isHugePageSupported returns true if the default hugepagesize on host is 2Mi (i.e. 2048 kB) -func isHugePageSupported() bool { - outData, err := exec.Command("/bin/sh", "-c", "cat /proc/meminfo | grep 'Hugepagesize:' | awk '{print $2}'").Output() - framework.ExpectNoError(err) - pageSize, err := strconv.Atoi(strings.TrimSpace(string(outData))) - framework.ExpectNoError(err) - return pageSize == 2048 -} - -// pollResourceAsString polls for a specified resource and capacity from node -func pollResourceAsString(f *framework.Framework, resourceName string) string { - node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), framework.TestContext.NodeName, metav1.GetOptions{}) - framework.ExpectNoError(err) - amount := amountOfResourceAsString(node, resourceName) - framework.Logf("amount of %v: %v", resourceName, amount) - return amount -} - -// amountOfResourceAsString returns the amount of resourceName advertised by a node -func amountOfResourceAsString(node *v1.Node, resourceName string) string { - val, ok := node.Status.Capacity[v1.ResourceName(resourceName)] - if !ok { - return "" +// isHugePageAvailable returns true if hugepages of the specified size is available on the host +func isHugePageAvailable(hugepagesSize int) bool { + path := fmt.Sprintf("%s-%dkB/%s", hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile) + if _, err := os.Stat(path); err != nil { + return false } - return val.String() + return true } -func runHugePagesTests(f *framework.Framework) { - ginkgo.It("should assign hugepages as expected based on the Pod spec", func() { - ginkgo.By("by running a G pod that requests hugepages") - pod := f.PodClient().Create(&v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod" + string(uuid.NewUUID()), - Namespace: f.Namespace.Name, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Image: imageutils.GetPauseImageName(), - Name: "container" + string(uuid.NewUUID()), - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceName("cpu"): resource.MustParse("10m"), - v1.ResourceName("memory"): resource.MustParse("100Mi"), - v1.ResourceName("hugepages-2Mi"): resource.MustParse("6Mi"), - }, - }, +func getHugepagesTestPod(f *framework.Framework, limits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "hugepages-", + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container" + string(uuid.NewUUID()), + Image: busyboxImage, + Resources: v1.ResourceRequirements{ + Limits: limits, }, + Command: []string{"sleep", "3600"}, + VolumeMounts: mounts, }, }, - }) - podUID := string(pod.UID) - ginkgo.By("checking if the expected hugetlb settings were applied") - verifyPod := makePodToVerifyHugePages("pod"+podUID, resource.MustParse("6Mi")) - f.PodClient().Create(verifyPod) - err := e2epod.WaitForPodSuccessInNamespace(f.ClientSet, verifyPod.Name, f.Namespace.Name) - framework.ExpectNoError(err) + Volumes: volumes, + }, + } +} + +// Serial because the test updates kubelet configuration. +var _ = SIGDescribe("HugePages [Serial] [Feature:HugePages][NodeSpecialFeature:HugePages]", func() { + f := framework.NewDefaultFramework("hugepages-test") + + ginkgo.It("should remove resources for huge page sizes no longer supported", func() { + ginkgo.By("mimicking support for 9Mi of 3Mi huge page memory by patching the node status") + patch := []byte(`[{"op": "add", "path": "/status/capacity/hugepages-3Mi", "value": "9Mi"}, {"op": "add", "path": "/status/allocatable/hugepages-3Mi", "value": "9Mi"}]`) + result := f.ClientSet.CoreV1().RESTClient().Patch(types.JSONPatchType).Resource("nodes").Name(framework.TestContext.NodeName).SubResource("status").Body(patch).Do(context.TODO()) + framework.ExpectNoError(result.Error(), "while patching") + + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), framework.TestContext.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err, "while getting node status") + + ginkgo.By("Verifying that the node now supports huge pages with size 3Mi") + value, ok := node.Status.Capacity["hugepages-3Mi"] + framework.ExpectEqual(ok, true, "capacity should contain resource hugepages-3Mi") + framework.ExpectEqual(value.String(), "9Mi", "huge pages with size 3Mi should be supported") + + ginkgo.By("restarting the node and verifying that huge pages with size 3Mi are not supported") + restartKubelet() + + ginkgo.By("verifying that the hugepages-3Mi resource no longer is present") + gomega.Eventually(func() bool { + node, err = f.ClientSet.CoreV1().Nodes().Get(context.TODO(), framework.TestContext.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err, "while getting node status") + _, isPresent := node.Status.Capacity["hugepages-3Mi"] + return isPresent + }, 30*time.Second, framework.Poll).Should(gomega.Equal(false)) }) ginkgo.It("should add resources for new huge page sizes on kubelet restart", func() { @@ -204,77 +237,240 @@ func runHugePagesTests(f *framework.Framework) { return isPresent }, 30*time.Second, framework.Poll).Should(gomega.Equal(true)) }) -} -// Serial because the test updates kubelet configuration. -var _ = SIGDescribe("HugePages [Serial] [Feature:HugePages][NodeFeature:HugePages]", func() { - f := framework.NewDefaultFramework("hugepages-test") + ginkgo.When("start the pod", func() { + var ( + testpod *v1.Pod + limits v1.ResourceList + mounts []v1.VolumeMount + volumes []v1.Volume + hugepages map[string]int + ) - ginkgo.It("should remove resources for huge page sizes no longer supported", func() { - ginkgo.By("mimicking support for 9Mi of 3Mi huge page memory by patching the node status") - patch := []byte(`[{"op": "add", "path": "/status/capacity/hugepages-3Mi", "value": "9Mi"}, {"op": "add", "path": "/status/allocatable/hugepages-3Mi", "value": "9Mi"}]`) - result := f.ClientSet.CoreV1().RESTClient().Patch(types.JSONPatchType).Resource("nodes").Name(framework.TestContext.NodeName).SubResource("status").Body(patch).Do(context.TODO()) - framework.ExpectNoError(result.Error(), "while patching") + setHugepages := func() { + for hugepagesResource, count := range hugepages { + size := resourceToSize[hugepagesResource] + ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size)) + if !isHugePageAvailable(size) { + e2eskipper.Skipf("skipping test because hugepages of size %d not supported", size) + return + } - node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), framework.TestContext.NodeName, metav1.GetOptions{}) - framework.ExpectNoError(err, "while getting node status") - - ginkgo.By("Verifying that the node now supports huge pages with size 3Mi") - value, ok := node.Status.Capacity["hugepages-3Mi"] - framework.ExpectEqual(ok, true, "capacity should contain resouce hugepages-3Mi") - framework.ExpectEqual(value.String(), "9Mi", "huge pages with size 3Mi should be supported") - - ginkgo.By("restarting the node and verifying that huge pages with size 3Mi are not supported") - restartKubelet() - - ginkgo.By("verifying that the hugepages-3Mi resource no longer is present") - gomega.Eventually(func() bool { - node, err = f.ClientSet.CoreV1().Nodes().Get(context.TODO(), framework.TestContext.NodeName, metav1.GetOptions{}) - framework.ExpectNoError(err, "while getting node status") - _, isPresent := node.Status.Capacity["hugepages-3Mi"] - return isPresent - }, 30*time.Second, framework.Poll).Should(gomega.Equal(false)) - }) - ginkgo.Context("With config updated with hugepages feature enabled", func() { - ginkgo.BeforeEach(func() { - ginkgo.By("verifying hugepages are supported") - if !isHugePageSupported() { - e2eskipper.Skipf("skipping test because hugepages are not supported") - return + ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size)) + gomega.Eventually(func() error { + if err := configureHugePages(size, count); err != nil { + return err + } + return nil + }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) } - ginkgo.By("configuring the host to reserve a number of pre-allocated hugepages") + } + + waitForHugepages := func() { + ginkgo.By("Waiting for hugepages resource to become available on the local node") gomega.Eventually(func() error { - err := configureHugePages() + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), framework.TestContext.NodeName, metav1.GetOptions{}) if err != nil { return err } + + for hugepagesResource, count := range hugepages { + capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)] + if !ok { + return fmt.Errorf("the node does not have the resource %s", hugepagesResource) + } + + size, succeed := capacity.AsInt64() + if !succeed { + return fmt.Errorf("failed to convert quantity to int64") + } + + expectedSize := count * resourceToSize[hugepagesResource] * 1024 + if size != int64(expectedSize) { + return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize) + } + } return nil - }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) - ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") - restartKubelet() - ginkgo.By("by waiting for hugepages resource to become available on the local node") - gomega.Eventually(func() string { - return pollResourceAsString(f, "hugepages-2Mi") - }, 30*time.Second, framework.Poll).Should(gomega.Equal("10Mi")) - }) + }, time.Minute, framework.Poll).Should(gomega.BeNil()) + } - runHugePagesTests(f) - - ginkgo.AfterEach(func() { + releaseHugepages := func() { ginkgo.By("Releasing hugepages") gomega.Eventually(func() error { - err := releaseHugePages() - if err != nil { - return err + for hugepagesResource := range hugepages { + command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile) + if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { + return err + } } return nil }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) - ginkgo.By("restarting kubelet to release hugepages") + } + + runHugePagesTests := func() { + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func() { + ginkgo.By("getting mounts for the test pod") + command := []string{"mount"} + out := f.ExecCommandInContainer(testpod.Name, testpod.Spec.Containers[0].Name, command...) + + for _, mount := range mounts { + ginkgo.By(fmt.Sprintf("checking that the hugetlb mount %s exists under the container", mount.MountPath)) + gomega.Expect(out).To(gomega.ContainSubstring(mount.MountPath)) + } + + for resourceName := range hugepages { + verifyPod := makePodToVerifyHugePages( + "pod"+string(testpod.UID), + testpod.Spec.Containers[0].Resources.Limits[v1.ResourceName(resourceName)], + resourceToCgroup[resourceName], + ) + ginkgo.By("checking if the expected hugetlb settings were applied") + f.PodClient().Create(verifyPod) + err := e2epod.WaitForPodSuccessInNamespace(f.ClientSet, verifyPod.Name, f.Namespace.Name) + gomega.Expect(err).To(gomega.BeNil()) + } + }) + } + + // setup + ginkgo.JustBeforeEach(func() { + setHugepages() + + ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") restartKubelet() - ginkgo.By("by waiting for hugepages resource to not appear available on the local node") - gomega.Eventually(func() string { - return pollResourceAsString(f, "hugepages-2Mi") - }, 30*time.Second, framework.Poll).Should(gomega.Equal("0")) + + waitForHugepages() + + pod := getHugepagesTestPod(f, limits, mounts, volumes) + + ginkgo.By("by running a guarantee pod that requests hugepages") + testpod = f.PodClient().CreateSync(pod) + }) + + // we should use JustAfterEach because framework will teardown the client under the AfterEach method + ginkgo.JustAfterEach(func() { + releaseHugepages() + + ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") + restartKubelet() + + waitForHugepages() + }) + + ginkgo.Context("with the resources requests that contain only one hugepages resource ", func() { + ginkgo.Context("with the backward compatible API", func() { + ginkgo.BeforeEach(func() { + limits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages", + MountPath: "/hugepages", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages, + }, + }, + }, + } + hugepages = map[string]int{hugepagesResourceName2Mi: 5} + }) + // run tests + runHugePagesTests() + }) + + ginkgo.Context("with the new API", func() { + ginkgo.BeforeEach(func() { + limits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + } + hugepages = map[string]int{hugepagesResourceName2Mi: 5} + }) + + runHugePagesTests() + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{hugepagesResourceName2Mi: 0} + }) + }) + + ginkgo.Context("with the resources requests that contain multiple hugepages resources ", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + hugepagesResourceName1Gi: 1, + } + limits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + { + Name: "hugepages-1gi", + MountPath: "/hugepages-1Gi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + { + Name: "hugepages-1gi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages1Gi, + }, + }, + }, + } + }) + + runHugePagesTests() + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + hugepagesResourceName1Gi: 0, + } + }) }) }) })