Move pod level hugepage test from e2e to e2e node

This commit is contained in:
Kevin Torres 2025-03-09 05:09:58 +00:00
parent 98e362c049
commit 7b38bff6ec
4 changed files with 673 additions and 455 deletions

View File

@ -29,17 +29,12 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
v1resource "k8s.io/kubernetes/pkg/api/v1/resource"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
kubecm "k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
utils "k8s.io/kubernetes/test/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
)
@ -48,14 +43,9 @@ const (
cgroupv2CPUWeight string = "cpu.weight"
cgroupv2CPULimit string = "cpu.max"
cgroupv2MemLimit string = "memory.max"
cgroupv2HugeTLBPrefix string = "hugetlb"
cgroupv2HugeTLBRsvd string = "rsvd"
cgroupFsPath string = "/sys/fs/cgroup"
mountPath string = "/sysfscgroup"
CPUPeriod string = "100000"
cgroupFsPath string = "/sys/fs/cgroup"
CPUPeriod string = "100000"
mountPath string = "/sysfscgroup"
)
var (
@ -79,7 +69,6 @@ var _ = SIGDescribe("Pod Level Resources", framework.WithSerial(), feature.PodLe
e2eskipper.Skipf("not supported on cgroupv1 -- skipping")
}
})
podLevelResourcesTests(f)
})
@ -115,7 +104,7 @@ func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool {
func makeObjectMetadata(name, namespace string) metav1.ObjectMeta {
return metav1.ObjectMeta{
Name: name, Namespace: namespace,
Name: "testpod", Namespace: namespace,
Labels: map[string]string{"time": strconv.Itoa(time.Now().Nanosecond())},
}
}
@ -124,16 +113,11 @@ type containerInfo struct {
Name string
Resources *resourceInfo
}
type resourceInfo struct {
CPUReq string
CPULim string
MemReq string
MemLim string
HugePagesReq2Mi string
HugePagesLim2Mi string
HugePagesReq1Gi string
HugePagesLim1Gi string
CPUReq string
CPULim string
MemReq string
MemLim string
}
func makeContainer(info containerInfo) v1.Container {
@ -156,7 +140,7 @@ func makeContainer(info containerInfo) v1.Container {
func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements {
var res v1.ResourceRequirements
if info != nil {
if info.CPUReq != "" || info.MemReq != "" || info.HugePagesReq2Mi != "" || info.HugePagesReq1Gi != "" {
if info.CPUReq != "" || info.MemReq != "" {
res.Requests = make(v1.ResourceList)
}
if info.CPUReq != "" {
@ -165,14 +149,8 @@ func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements {
if info.MemReq != "" {
res.Requests[v1.ResourceMemory] = resource.MustParse(info.MemReq)
}
if info.HugePagesReq2Mi != "" {
res.Requests[v1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse(info.HugePagesReq2Mi)
}
if info.HugePagesReq1Gi != "" {
res.Requests[v1.ResourceHugePagesPrefix+"1Gi"] = resource.MustParse(info.HugePagesReq1Gi)
}
if info.CPULim != "" || info.MemLim != "" || info.HugePagesLim2Mi != "" || info.HugePagesLim1Gi != "" {
if info.CPULim != "" || info.MemLim != "" {
res.Limits = make(v1.ResourceList)
}
if info.CPULim != "" {
@ -181,12 +159,6 @@ func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements {
if info.MemLim != "" {
res.Limits[v1.ResourceMemory] = resource.MustParse(info.MemLim)
}
if info.HugePagesLim2Mi != "" {
res.Limits[v1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse(info.HugePagesLim2Mi)
}
if info.HugePagesLim1Gi != "" {
res.Limits[v1.ResourceHugePagesPrefix+"1Gi"] = resource.MustParse(info.HugePagesLim1Gi)
}
}
return res
}
@ -239,7 +211,7 @@ func verifyQoS(gotPod v1.Pod, expectedQoS v1.PodQOSClass) {
}
// TODO(ndixita): dedup the conversion logic in pod resize test and move to helpers/utils.
func verifyPodCgroups(f *framework.Framework, pod *v1.Pod, info *resourceInfo) error {
func verifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, info *resourceInfo) error {
ginkgo.GinkgoHelper()
cmd := fmt.Sprintf("find %s -name '*%s*'", mountPath, strings.ReplaceAll(string(pod.UID), "-", "_"))
framework.Logf("Namespace %s Pod %s - looking for Pod cgroup directory path: %q", f.Namespace, pod.Name, cmd)
@ -275,70 +247,6 @@ func verifyPodCgroups(f *framework.Framework, pod *v1.Pod, info *resourceInfo) e
if err != nil {
errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
}
// Verify cgroup limits for all the hugepage sizes in the pod
for resourceName, resourceAmount := range expectedResources.Limits {
if !v1resource.IsHugePageResourceName(resourceName) {
continue
}
pageSize, err := v1helper.HugePageSizeFromResourceName(resourceName)
if err != nil {
errs = append(errs, fmt.Errorf("encountered error while obtaining hugepage size: %w", err))
}
sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value())
if err != nil {
errs = append(errs, fmt.Errorf("encountered error while obtaining hugepage unit size: %w", err))
}
hugepageCgroupv2Limits := []string{
fmt.Sprintf("%s.%s.max", cgroupv2HugeTLBPrefix, sizeString),
fmt.Sprintf("%s.%s.%s.max", cgroupv2HugeTLBPrefix, sizeString, cgroupv2HugeTLBRsvd),
}
expectedHugepageLim := strconv.FormatInt(resourceAmount.Value(), 10)
for _, hugepageCgroupv2Limit := range hugepageCgroupv2Limits {
hugepageLimCgPath := fmt.Sprintf("%s/%s", podCgPath, hugepageCgroupv2Limit)
err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, hugepageLimCgPath, expectedHugepageLim)
if err != nil {
errs = append(errs, fmt.Errorf("failed to verify hugepage limit cgroup value: %w, path: %s", err, hugepageLimCgPath))
}
}
}
return utilerrors.NewAggregate(errs)
}
func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error {
var errs []error
for _, container := range pod.Spec.Containers {
if pod.Spec.Resources == nil {
continue
}
if pod.Spec.Resources.Limits.Memory() != nil && container.Resources.Limits.Memory() == nil {
expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10)
err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit)
if err != nil {
errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
}
}
if pod.Spec.Resources.Limits.Cpu() != nil && container.Resources.Limits.Cpu() == nil {
cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod)
expectedCPULimit := strconv.FormatInt(cpuQuota, 10)
expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod)
err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit)
if err != nil {
errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err))
}
}
// TODO(KevinTMtz) - Check for all hugepages for the pod, for this is
// required to enabled the Containerd Cgroup value, because if not, HugeTLB
// cgroup values will be just set to max
}
return utilerrors.NewAggregate(errs)
}
@ -349,7 +257,7 @@ func podLevelResourcesTests(f *framework.Framework) {
// and limits for the pod. If pod-level resource specifications
// are specified, totalPodResources is equal to pod-level resources.
// Otherwise, it is calculated by aggregating resource requests and
// limits from all containers within the pod.
// limits from all containers within the pod..
totalPodResources *resourceInfo
}
@ -358,7 +266,6 @@ func podLevelResourcesTests(f *framework.Framework) {
podResources *resourceInfo
containers []containerInfo
expected expectedPodConfig
hugepages map[string]int
}
tests := []testCase{
@ -442,108 +349,10 @@ func podLevelResourcesTests(f *framework.Framework) {
totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
},
},
{
name: "Guaranteed QoS pod hugepages, no container resources, single page size",
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"},
containers: []containerInfo{{Name: "c1"}, {Name: "c2"}},
expected: expectedPodConfig{
qos: v1.PodQOSGuaranteed,
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi"},
},
hugepages: map[string]int{
v1.ResourceHugePagesPrefix + "2Mi": 5,
},
},
{
name: "Burstable QoS pod hugepages, container resources, single page size",
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"},
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi"}}, {Name: "c2"}},
expected: expectedPodConfig{
qos: v1.PodQOSBurstable,
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi"},
},
hugepages: map[string]int{
v1.ResourceHugePagesPrefix + "2Mi": 5,
},
},
{
name: "Burstable QoS pod hugepages, container resources, single page size, pod level does not specify hugepages",
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi"}}, {Name: "c2"}},
expected: expectedPodConfig{
qos: v1.PodQOSBurstable,
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "4Mi", HugePagesLim2Mi: "4Mi"},
},
hugepages: map[string]int{
v1.ResourceHugePagesPrefix + "2Mi": 2,
},
},
{
name: "Guaranteed QoS pod hugepages, no container resources, multiple page size",
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi", HugePagesLim1Gi: "1Gi"},
containers: []containerInfo{{Name: "c1"}, {Name: "c2"}},
expected: expectedPodConfig{
qos: v1.PodQOSGuaranteed,
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"},
},
hugepages: map[string]int{
v1.ResourceHugePagesPrefix + "2Mi": 5,
v1.ResourceHugePagesPrefix + "1Gi": 1,
},
},
{
name: "Burstable QoS pod hugepages, container resources, multiple page size",
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi", HugePagesLim1Gi: "1Gi"},
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}},
expected: expectedPodConfig{
qos: v1.PodQOSBurstable,
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"},
},
hugepages: map[string]int{
v1.ResourceHugePagesPrefix + "2Mi": 5,
v1.ResourceHugePagesPrefix + "1Gi": 1,
},
},
{
name: "Burstable QoS pod hugepages, container resources, multiple page size, pod level does not specify hugepages",
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}},
expected: expectedPodConfig{
qos: v1.PodQOSBurstable,
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "4Mi", HugePagesLim2Mi: "4Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"},
},
hugepages: map[string]int{
v1.ResourceHugePagesPrefix + "2Mi": 2,
v1.ResourceHugePagesPrefix + "1Gi": 1,
},
},
{
name: "Burstable QoS pod hugepages, container resources, different page size between pod and container level",
podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"},
containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}},
expected: expectedPodConfig{
qos: v1.PodQOSBurstable,
totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"},
},
hugepages: map[string]int{
v1.ResourceHugePagesPrefix + "2Mi": 5,
v1.ResourceHugePagesPrefix + "1Gi": 1,
},
},
}
for _, tc := range tests {
ginkgo.It(tc.name, func(ctx context.Context) {
// Pre-allocate hugepages in the node
if tc.hugepages != nil {
utils.SetHugepages(ctx, tc.hugepages)
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
utils.RestartKubelet(ctx, false)
utils.WaitForHugepages(ctx, f, tc.hugepages)
}
podMetadata := makeObjectMetadata("testpod", f.Namespace.Name)
testPod := makePod(&podMetadata, tc.podResources, tc.containers)
@ -558,7 +367,7 @@ func podLevelResourcesTests(f *framework.Framework) {
verifyQoS(*pod, tc.expected.qos)
ginkgo.By("verifying pod cgroup values")
err := verifyPodCgroups(f, pod, tc.expected.totalPodResources)
err := verifyPodCgroups(ctx, f, pod, tc.expected.totalPodResources)
framework.ExpectNoError(err, "failed to verify pod's cgroup values: %v", err)
ginkgo.By("verifying containers cgroup limits are same as pod container's cgroup limits")
@ -568,16 +377,32 @@ func podLevelResourcesTests(f *framework.Framework) {
ginkgo.By("deleting pods")
delErr := e2epod.DeletePodWithWait(ctx, f.ClientSet, pod)
framework.ExpectNoError(delErr, "failed to delete pod %s", delErr)
// Release pre-allocated hugepages
if tc.hugepages != nil {
utils.ReleaseHugepages(ctx, tc.hugepages)
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
utils.RestartKubelet(ctx, true)
utils.WaitForHugepages(ctx, f, tc.hugepages)
}
})
}
}
func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error {
var errs []error
for _, container := range pod.Spec.Containers {
if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Memory() != nil &&
container.Resources.Limits.Memory() == nil {
expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10)
err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit)
if err != nil {
errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
}
}
if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Cpu() != nil &&
container.Resources.Limits.Cpu() == nil {
cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod)
expectedCPULimit := strconv.FormatInt(cpuQuota, 10)
expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod)
err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit)
if err != nil {
errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err))
}
}
}
return utilerrors.NewAggregate(errs)
}

View File

@ -19,6 +19,10 @@ package e2enode
import (
"context"
"fmt"
"os"
"os/exec"
"strconv"
"strings"
"time"
"github.com/onsi/ginkgo/v2"
@ -29,11 +33,12 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
testutils "k8s.io/kubernetes/test/utils"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
admissionapi "k8s.io/pod-security-admission/api"
)
@ -115,8 +120,66 @@ func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity,
return pod
}
func getHugepagesTestPod(f *framework.Framework, limits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod {
return &v1.Pod{
// configureHugePages attempts to allocate hugepages of the specified size
func configureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error {
// Compact memory to make bigger contiguous blocks of memory available
// before allocating huge pages.
// https://www.kernel.org/doc/Documentation/sysctl/vm.txt
if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil {
if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil {
return err
}
}
// e.g. hugepages/hugepages-2048kB/nr_hugepages
hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile)
// e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix)
if numaNodeID != nil {
// e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix)
}
// Reserve number of hugepages
// e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile)
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
return err
}
// verify that the number of hugepages was updated
// e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages"
command = fmt.Sprintf("cat %s", hugepagesFile)
outData, err := exec.Command("/bin/sh", "-c", command).Output()
if err != nil {
return err
}
numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData)))
if err != nil {
return err
}
framework.Logf("Hugepages total is set to %v", numHugePages)
if numHugePages == hugepagesCount {
return nil
}
return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages)
}
// isHugePageAvailable returns true if hugepages of the specified size is available on the host
func isHugePageAvailable(hugepagesSize int) bool {
path := fmt.Sprintf("%s-%dkB/%s", hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile)
if _, err := os.Stat(path); err != nil {
return false
}
return true
}
func getHugepagesTestPod(f *framework.Framework, podLimits v1.ResourceList, containerLimits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "hugepages-",
Namespace: f.Namespace.Name,
@ -124,18 +187,110 @@ func getHugepagesTestPod(f *framework.Framework, limits v1.ResourceList, mounts
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "container" + string(uuid.NewUUID()),
Image: busyboxImage,
Resources: v1.ResourceRequirements{
Limits: limits,
},
Name: "container" + string(uuid.NewUUID()),
Image: busyboxImage,
Command: []string{"sleep", "3600"},
VolumeMounts: mounts,
Resources: v1.ResourceRequirements{
Limits: containerLimits,
},
},
},
Volumes: volumes,
},
}
if podLimits != nil {
pod.Spec.Resources = &v1.ResourceRequirements{
Limits: podLimits,
}
}
return pod
}
func setHugepages(ctx context.Context, hugepages map[string]int) {
for hugepagesResource, count := range hugepages {
size := resourceToSize[hugepagesResource]
ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size))
if !isHugePageAvailable(size) {
e2eskipper.Skipf("skipping test because hugepages of size %d not supported", size)
return
}
ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size))
gomega.Eventually(ctx, func() error {
if err := configureHugePages(size, count, nil); err != nil {
return err
}
return nil
}, 30*time.Second, framework.Poll).Should(gomega.Succeed())
}
}
func waitForHugepages(f *framework.Framework, ctx context.Context, hugepages map[string]int) {
ginkgo.By("Waiting for hugepages resource to become available on the local node")
gomega.Eventually(ctx, func(ctx context.Context) error {
node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{})
if err != nil {
return err
}
for hugepagesResource, count := range hugepages {
capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)]
if !ok {
return fmt.Errorf("the node does not have the resource %s", hugepagesResource)
}
size, succeed := capacity.AsInt64()
if !succeed {
return fmt.Errorf("failed to convert quantity to int64")
}
expectedSize := count * resourceToSize[hugepagesResource] * 1024
if size != int64(expectedSize) {
return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize)
}
}
return nil
}, time.Minute, framework.Poll).Should(gomega.Succeed())
}
func releaseHugepages(ctx context.Context, hugepages map[string]int) {
ginkgo.By("Releasing hugepages")
gomega.Eventually(ctx, func() error {
for hugepagesResource := range hugepages {
command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile)
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
return err
}
}
return nil
}, 30*time.Second, framework.Poll).Should(gomega.Succeed())
}
func runHugePagesTests(f *framework.Framework, ctx context.Context, testpod *v1.Pod, expectedHugepageLimits v1.ResourceList, mounts []v1.VolumeMount, hugepages map[string]int) {
ginkgo.By("getting mounts for the test pod")
command := []string{"mount"}
out := e2epod.ExecCommandInContainer(f, testpod.Name, testpod.Spec.Containers[0].Name, command...)
for _, mount := range mounts {
ginkgo.By(fmt.Sprintf("checking that the hugetlb mount %s exists under the container", mount.MountPath))
gomega.Expect(out).To(gomega.ContainSubstring(mount.MountPath))
}
for resourceName := range hugepages {
verifyPod := makePodToVerifyHugePages(
"pod"+string(testpod.UID),
expectedHugepageLimits[v1.ResourceName(resourceName)],
resourceToCgroup[resourceName],
)
ginkgo.By("checking if the expected hugetlb settings were applied")
e2epod.NewPodClient(f).Create(ctx, verifyPod)
err := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, verifyPod.Name, f.Namespace.Name)
framework.ExpectNoError(err)
}
}
// Serial because the test updates kubelet configuration.
@ -193,48 +348,24 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
ginkgo.When("start the pod", func() {
var (
testpod *v1.Pod
limits v1.ResourceList
mounts []v1.VolumeMount
volumes []v1.Volume
hugepages map[string]int
testpod *v1.Pod
expectedHugepageLimits v1.ResourceList
containerLimits v1.ResourceList
mounts []v1.VolumeMount
volumes []v1.Volume
hugepages map[string]int
)
runHugePagesTests := func() {
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
ginkgo.By("getting mounts for the test pod")
command := []string{"mount"}
out := e2epod.ExecCommandInContainer(f, testpod.Name, testpod.Spec.Containers[0].Name, command...)
for _, mount := range mounts {
ginkgo.By(fmt.Sprintf("checking that the hugetlb mount %s exists under the container", mount.MountPath))
gomega.Expect(out).To(gomega.ContainSubstring(mount.MountPath))
}
for resourceName := range hugepages {
verifyPod := makePodToVerifyHugePages(
"pod"+string(testpod.UID),
testpod.Spec.Containers[0].Resources.Limits[v1.ResourceName(resourceName)],
resourceToCgroup[resourceName],
)
ginkgo.By("checking if the expected hugetlb settings were applied")
e2epod.NewPodClient(f).Create(ctx, verifyPod)
err := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, verifyPod.Name, f.Namespace.Name)
framework.ExpectNoError(err)
}
})
}
// setup
ginkgo.JustBeforeEach(func(ctx context.Context) {
testutils.SetHugepages(ctx, hugepages)
setHugepages(ctx, hugepages)
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
restartKubelet(ctx, true)
testutils.WaitForHugepages(ctx, f, hugepages)
waitForHugepages(f, ctx, hugepages)
pod := getHugepagesTestPod(f, limits, mounts, volumes)
pod := getHugepagesTestPod(f, nil, containerLimits, mounts, volumes)
ginkgo.By("by running a test pod that requests hugepages")
testpod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
@ -245,18 +376,21 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
ginkgo.By(fmt.Sprintf("deleting test pod %s", testpod.Name))
e2epod.NewPodClient(f).DeleteSync(ctx, testpod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete)
testutils.ReleaseHugepages(ctx, hugepages)
releaseHugepages(ctx, hugepages)
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
restartKubelet(ctx, true)
testutils.WaitForHugepages(ctx, f, hugepages)
waitForHugepages(f, ctx, hugepages)
})
ginkgo.Context("with the resources requests that contain only one hugepages resource ", func() {
ginkgo.Context("with the backward compatible API", func() {
ginkgo.BeforeEach(func() {
limits = v1.ResourceList{
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
}
containerLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
@ -280,12 +414,17 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
hugepages = map[string]int{hugepagesResourceName2Mi: 5}
})
// run tests
runHugePagesTests()
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
})
ginkgo.Context("with the new API", func() {
ginkgo.BeforeEach(func() {
limits = v1.ResourceList{
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
}
containerLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
@ -309,7 +448,9 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
hugepages = map[string]int{hugepagesResourceName2Mi: 5}
})
runHugePagesTests()
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
})
ginkgo.JustAfterEach(func() {
@ -323,7 +464,11 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
hugepagesResourceName2Mi: 5,
hugepagesResourceName1Gi: 1,
}
limits = v1.ResourceList{
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
containerLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
@ -359,7 +504,443 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func
}
})
runHugePagesTests()
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
ginkgo.JustAfterEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 0,
hugepagesResourceName1Gi: 0,
}
})
})
})
})
// Serial because the test updates kubelet configuration.
var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), feature.PodLevelResources, func() {
f := framework.NewDefaultFramework("pod-level-hugepages-resources")
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
ginkgo.When("pod level resources", func() {
var (
testpod *v1.Pod
expectedHugepageLimits v1.ResourceList
podLimits v1.ResourceList
containerLimits v1.ResourceList
mounts []v1.VolumeMount
volumes []v1.Volume
hugepages map[string]int
)
// setup
ginkgo.JustBeforeEach(func(ctx context.Context) {
e2eskipper.SkipUnlessFeatureGateEnabled(features.PodLevelResources)
setHugepages(ctx, hugepages)
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
restartKubelet(ctx, true)
waitForHugepages(f, ctx, hugepages)
pod := getHugepagesTestPod(f, podLimits, containerLimits, mounts, volumes)
ginkgo.By("by running a test pod that requests hugepages")
testpod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
framework.Logf("Test pod name: %s", testpod.Name)
})
// we should use JustAfterEach because framework will teardown the client under the AfterEach method
ginkgo.JustAfterEach(func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("deleting test pod %s", testpod.Name))
e2epod.NewPodClient(f).DeleteSync(ctx, testpod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete)
releaseHugepages(ctx, hugepages)
ginkgo.By("restarting kubelet to pick up pre-allocated hugepages")
restartKubelet(ctx, true)
waitForHugepages(f, ctx, hugepages)
})
ginkgo.Context("pod hugepages, no container hugepages, single page size", func() {
ginkgo.BeforeEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 5,
}
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
}
podLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
}
containerLimits = v1.ResourceList{}
mounts = []v1.VolumeMount{
{
Name: "hugepages-2mi",
MountPath: "/hugepages-2Mi",
},
}
volumes = []v1.Volume{
{
Name: "hugepages-2mi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages2Mi,
},
},
},
}
})
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
ginkgo.JustAfterEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 0,
}
})
})
ginkgo.Context("pod hugepages, container hugepages, single page size", func() {
ginkgo.BeforeEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 5,
}
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
}
podLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
}
containerLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
}
mounts = []v1.VolumeMount{
{
Name: "hugepages-2mi",
MountPath: "/hugepages-2Mi",
},
}
volumes = []v1.Volume{
{
Name: "hugepages-2mi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages2Mi,
},
},
},
}
})
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
ginkgo.JustAfterEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 0,
}
})
})
ginkgo.Context("no pod hugepages, container hugepages, single page size", func() {
ginkgo.BeforeEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 5,
}
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
}
podLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
}
containerLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
}
mounts = []v1.VolumeMount{
{
Name: "hugepages-2mi",
MountPath: "/hugepages-2Mi",
},
}
volumes = []v1.Volume{
{
Name: "hugepages-2mi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages2Mi,
},
},
},
}
})
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
ginkgo.JustAfterEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 0,
}
})
})
ginkgo.Context("pod hugepages, no container hugepages, multiple page size", func() {
ginkgo.BeforeEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 5,
hugepagesResourceName1Gi: 1,
}
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
podLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
containerLimits = v1.ResourceList{}
mounts = []v1.VolumeMount{
{
Name: "hugepages-2mi",
MountPath: "/hugepages-2Mi",
},
{
Name: "hugepages-1gi",
MountPath: "/hugepages-1Gi",
},
}
volumes = []v1.Volume{
{
Name: "hugepages-2mi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages2Mi,
},
},
},
{
Name: "hugepages-1gi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages1Gi,
},
},
},
}
})
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
ginkgo.JustAfterEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 0,
hugepagesResourceName1Gi: 0,
}
})
})
ginkgo.Context("pod hugepages, container hugepages, multiple page size", func() {
ginkgo.BeforeEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 5,
hugepagesResourceName1Gi: 1,
}
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
podLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
containerLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
mounts = []v1.VolumeMount{
{
Name: "hugepages-2mi",
MountPath: "/hugepages-2Mi",
},
{
Name: "hugepages-1gi",
MountPath: "/hugepages-1Gi",
},
}
volumes = []v1.Volume{
{
Name: "hugepages-2mi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages2Mi,
},
},
},
{
Name: "hugepages-1gi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages1Gi,
},
},
},
}
})
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
ginkgo.JustAfterEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 0,
hugepagesResourceName1Gi: 0,
}
})
})
ginkgo.Context("no pod hugepages, container hugepages, multiple page size", func() {
ginkgo.BeforeEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 5,
hugepagesResourceName1Gi: 1,
}
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
podLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
}
containerLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("4Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
mounts = []v1.VolumeMount{
{
Name: "hugepages-2mi",
MountPath: "/hugepages-2Mi",
},
{
Name: "hugepages-1gi",
MountPath: "/hugepages-1Gi",
},
}
volumes = []v1.Volume{
{
Name: "hugepages-2mi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages2Mi,
},
},
},
{
Name: "hugepages-1gi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages1Gi,
},
},
},
}
})
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
ginkgo.JustAfterEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 0,
hugepagesResourceName1Gi: 0,
}
})
})
ginkgo.Context("pod hugepages, container hugepages, different page size between pod and container level", func() {
ginkgo.BeforeEach(func() {
hugepages = map[string]int{
hugepagesResourceName2Mi: 5,
hugepagesResourceName1Gi: 1,
}
expectedHugepageLimits = v1.ResourceList{
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
podLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName2Mi: resource.MustParse("6Mi"),
}
containerLimits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
hugepagesResourceName1Gi: resource.MustParse("1Gi"),
}
mounts = []v1.VolumeMount{
{
Name: "hugepages-2mi",
MountPath: "/hugepages-2Mi",
},
{
Name: "hugepages-1gi",
MountPath: "/hugepages-1Gi",
},
}
volumes = []v1.Volume{
{
Name: "hugepages-2mi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages2Mi,
},
},
},
{
Name: "hugepages-1gi",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
Medium: mediumHugepages1Gi,
},
},
},
}
})
ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) {
runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages)
})
ginkgo.JustAfterEach(func() {
hugepages = map[string]int{

View File

@ -42,7 +42,6 @@ import (
"k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
testutils "k8s.io/kubernetes/test/utils"
admissionapi "k8s.io/pod-security-admission/api"
"k8s.io/utils/cpuset"
"k8s.io/utils/pointer"
@ -315,7 +314,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With
}
if is2MiHugepagesSupported == nil {
is2MiHugepagesSupported = pointer.BoolPtr(testutils.IsHugePageAvailable(hugepagesSize2M))
is2MiHugepagesSupported = pointer.BoolPtr(isHugePageAvailable(hugepagesSize2M))
}
if len(allNUMANodes) == 0 {
@ -326,7 +325,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With
if *is2MiHugepagesSupported {
ginkgo.By("Configuring hugepages")
gomega.Eventually(ctx, func() error {
return testutils.ConfigureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0))
return configureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0))
}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
}
})
@ -359,7 +358,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With
ginkgo.By("Releasing allocated hugepages")
gomega.Eventually(ctx, func() error {
// configure hugepages on the NUMA node 0 to avoid hugepages split across NUMA nodes
return testutils.ConfigureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0))
return configureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0))
}, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages")
}
})

View File

@ -16,39 +16,7 @@ limitations under the License.
package utils
import (
"context"
"fmt"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
)
const (
hugepagesCapacityFile = "nr_hugepages"
hugepagesDirPrefix = "/sys/kernel/mm/hugepages/hugepages"
hugepagesSize2M = 2048
hugepagesSize1G = 1048576
)
var (
resourceToSize = map[string]int{
v1.ResourceHugePagesPrefix + "2Mi": hugepagesSize2M,
v1.ResourceHugePagesPrefix + "1Gi": hugepagesSize1G,
}
)
import v1 "k8s.io/api/core/v1"
// GetNodeCondition extracts the provided condition from the given status and returns that.
// Returns nil and -1 if the condition is not present, and the index of the located condition.
@ -63,158 +31,3 @@ func GetNodeCondition(status *v1.NodeStatus, conditionType v1.NodeConditionType)
}
return -1, nil
}
func SetHugepages(ctx context.Context, hugepages map[string]int) {
for hugepagesResource, count := range hugepages {
size := resourceToSize[hugepagesResource]
ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size))
if !IsHugePageAvailable(size) {
skipf("skipping test because hugepages of size %d not supported", size)
return
}
ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size))
gomega.Eventually(ctx, func() error {
if err := ConfigureHugePages(size, count, nil); err != nil {
return err
}
return nil
}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
}
}
func IsHugePageAvailable(size int) bool {
// e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
hugepagesFile := fmt.Sprintf("/sys/kernel/mm/hugepages/hugepages-%dkB/nr_hugepages", size)
if _, err := os.Stat(hugepagesFile); err != nil {
framework.Logf("Hugepages file %s not found: %v", hugepagesFile, err)
return false
}
return true
}
// configureHugePages attempts to allocate hugepages of the specified size
func ConfigureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error {
// Compact memory to make bigger contiguous blocks of memory available
// before allocating huge pages.
// https://www.kernel.org/doc/Documentation/sysctl/vm.txt
if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil {
if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil {
return err
}
}
// e.g. hugepages/hugepages-2048kB/nr_hugepages
hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile)
// e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix)
if numaNodeID != nil {
// e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix)
}
// Reserve number of hugepages
// e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile)
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
return err
}
// verify that the number of hugepages was updated
// e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages"
command = fmt.Sprintf("cat %s", hugepagesFile)
outData, err := exec.Command("/bin/sh", "-c", command).Output()
if err != nil {
return err
}
numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData)))
if err != nil {
return err
}
framework.Logf("Hugepages total is set to %v", numHugePages)
if numHugePages == hugepagesCount {
return nil
}
return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages)
}
// TODO(KevinTMtz) - Deduplicate from test/e2e_node/util.go:restartKubelet
func RestartKubelet(ctx context.Context, running bool) {
kubeletServiceName := FindKubeletServiceName(running)
// reset the kubelet service start-limit-hit
stdout, err := exec.CommandContext(ctx, "sudo", "systemctl", "reset-failed", kubeletServiceName).CombinedOutput()
framework.ExpectNoError(err, "Failed to reset kubelet start-limit-hit with systemctl: %v, %s", err, string(stdout))
stdout, err = exec.CommandContext(ctx, "sudo", "systemctl", "restart", kubeletServiceName).CombinedOutput()
framework.ExpectNoError(err, "Failed to restart kubelet with systemctl: %v, %s", err, string(stdout))
}
func FindKubeletServiceName(running bool) string {
cmdLine := []string{
"systemctl", "list-units", "*kubelet*",
}
if running {
cmdLine = append(cmdLine, "--state=running")
}
stdout, err := exec.Command("sudo", cmdLine...).CombinedOutput()
framework.ExpectNoError(err)
regex := regexp.MustCompile("(kubelet-\\w+)")
matches := regex.FindStringSubmatch(string(stdout))
gomega.Expect(matches).ToNot(gomega.BeEmpty(), "Found more than one kubelet service running: %q", stdout)
kubeletServiceName := matches[0]
framework.Logf("Get running kubelet with systemctl: %v, %v", string(stdout), kubeletServiceName)
return kubeletServiceName
}
func WaitForHugepages(ctx context.Context, f *framework.Framework, hugepages map[string]int) {
ginkgo.By("Waiting for hugepages resource to become available on the local node")
gomega.Eventually(ctx, func(ctx context.Context) error {
node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{})
if err != nil {
return err
}
for hugepagesResource, count := range hugepages {
capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)]
if !ok {
return fmt.Errorf("the node does not have the resource %s", hugepagesResource)
}
size, succeed := capacity.AsInt64()
if !succeed {
return fmt.Errorf("failed to convert quantity to int64")
}
expectedSize := count * resourceToSize[hugepagesResource] * 1024
if size != int64(expectedSize) {
return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize)
}
}
return nil
}, time.Minute, framework.Poll).Should(gomega.BeNil())
}
func ReleaseHugepages(ctx context.Context, hugepages map[string]int) {
ginkgo.By("Releasing hugepages")
gomega.Eventually(ctx, func() error {
for hugepagesResource := range hugepages {
command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile)
if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil {
return err
}
}
return nil
}, 30*time.Second, framework.Poll).Should(gomega.BeNil())
}
// TODO(KevinTMtz) - Deduplicate from test/e2e/framework/skipper/skipper.go:Skipf
func skipf(format string, args ...any) {
msg := fmt.Sprintf(format, args...)
ginkgo.Skip(msg, 2)
panic("unreachable")
}