From e5020285fa27b0d3b2587d2167f89ee05e86a8d4 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 19 Feb 2025 19:29:41 +0000 Subject: [PATCH 1/8] Hugepages to pod level supported resources --- staging/src/k8s.io/component-helpers/resource/helpers.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/component-helpers/resource/helpers.go b/staging/src/k8s.io/component-helpers/resource/helpers.go index 683dc3ad8b8..291510fc920 100644 --- a/staging/src/k8s.io/component-helpers/resource/helpers.go +++ b/staging/src/k8s.io/component-helpers/resource/helpers.go @@ -17,6 +17,8 @@ limitations under the License. package resource import ( + "strings" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" ) @@ -56,14 +58,14 @@ type PodResourcesOptions struct { var supportedPodLevelResources = sets.New(v1.ResourceCPU, v1.ResourceMemory) func SupportedPodLevelResources() sets.Set[v1.ResourceName] { - return supportedPodLevelResources + return supportedPodLevelResources.Clone().Insert(v1.ResourceHugePagesPrefix) } // IsSupportedPodLevelResources checks if a given resource is supported by pod-level // resource management through the PodLevelResources feature. Returns true if // the resource is supported. func IsSupportedPodLevelResource(name v1.ResourceName) bool { - return supportedPodLevelResources.Has(name) + return supportedPodLevelResources.Has(name) || strings.HasPrefix(string(name), v1.ResourceHugePagesPrefix) } // IsPodLevelResourcesSet check if PodLevelResources pod-level resources are set. From aa17082a160f1f4773f26fc34ed80b419ddfe3b7 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 19 Feb 2025 19:36:05 +0000 Subject: [PATCH 2/8] Default pod level hugepage limits --- pkg/apis/core/v1/defaults.go | 49 +++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/pkg/apis/core/v1/defaults.go b/pkg/apis/core/v1/defaults.go index e66de8bb432..b2d81d6fa87 100644 --- a/pkg/apis/core/v1/defaults.go +++ b/pkg/apis/core/v1/defaults.go @@ -27,6 +27,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" resourcehelper "k8s.io/component-helpers/resource" "k8s.io/kubernetes/pkg/api/v1/service" + corev1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/parsers" ) @@ -199,6 +200,7 @@ func SetDefaults_Pod(obj *v1.Pod) { // Pod Requests default values must be applied after container-level default values // have been populated. if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources) { + defaultHugePagePodLimits(obj) defaultPodRequests(obj) } @@ -456,7 +458,9 @@ func defaultPodRequests(obj *v1.Pod) { // PodLevelResources feature) and pod-level requests are not set, the pod-level requests // default to the effective requests of all the containers for that resource. for key, aggrCtrLim := range aggrCtrReqs { - if _, exists := podReqs[key]; !exists && resourcehelper.IsSupportedPodLevelResource(key) { + // Defaulting for pod level hugepages requests takes them directly from the pod limit, + // hugepages cannot be overcommited and must have the limit, so we skip them here. + if _, exists := podReqs[key]; !exists && resourcehelper.IsSupportedPodLevelResource(key) && !corev1helper.IsHugePageResourceName(key) { podReqs[key] = aggrCtrLim.DeepCopy() } } @@ -464,6 +468,8 @@ func defaultPodRequests(obj *v1.Pod) { // When no containers specify requests for a resource, the pod-level requests // will default to match the pod-level limits, if pod-level // limits exist for that resource. + // Defaulting for pod level hugepages requests is dependent on defaultHugePagePodLimits, + // if defaultHugePagePodLimits defined the limit, the request will be set here. for key, podLim := range obj.Spec.Resources.Limits { if _, exists := podReqs[key]; !exists && resourcehelper.IsSupportedPodLevelResource(key) { podReqs[key] = podLim.DeepCopy() @@ -476,3 +482,44 @@ func defaultPodRequests(obj *v1.Pod) { obj.Spec.Resources.Requests = podReqs } } + +// defaultHugePagePodLimits applies default values for pod-level limits, only when +// container hugepage limits are set, but not at pod level, in following +// scenario: +// 1. When at least one container (regular, init or sidecar) has hugepage +// limits set: +// The pod-level limit becomes equal to the aggregated hugepages limit of all +// the containers in the pod. +func defaultHugePagePodLimits(obj *v1.Pod) { + // We only populate defaults when the pod-level resources are partly specified already. + if obj.Spec.Resources == nil { + return + } + + if len(obj.Spec.Resources.Limits) == 0 && len(obj.Spec.Resources.Requests) == 0 { + return + } + + var podLims v1.ResourceList + podLims = obj.Spec.Resources.Limits + if podLims == nil { + podLims = make(v1.ResourceList) + } + + aggrCtrLims := resourcehelper.AggregateContainerLimits(obj, resourcehelper.PodResourcesOptions{}) + + // When containers specify limits for hugepages and pod-level limits are not + // set for that resource, the pod-level limit will default to the aggregated + // hugepages limit of all the containers. + for key, aggrCtrLim := range aggrCtrLims { + if _, exists := podLims[key]; !exists && resourcehelper.IsSupportedPodLevelResource(key) && corev1helper.IsHugePageResourceName(key) { + podLims[key] = aggrCtrLim.DeepCopy() + } + } + + // Only set pod-level resource limits in the PodSpec if the requirements map + // contains entries after collecting container-level limits and pod-level limits for hugepages. + if len(podLims) > 0 { + obj.Spec.Resources.Limits = podLims + } +} From e3f4c6e62abb536d5b996477fc9be4eeafc05d9f Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 19 Feb 2025 19:40:42 +0000 Subject: [PATCH 3/8] Containers with hugepage volume mounts with unset hugepage limits --- pkg/volume/emptydir/empty_dir.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go index a9128f661fa..451da5ed460 100644 --- a/pkg/volume/emptydir/empty_dir.go +++ b/pkg/volume/emptydir/empty_dir.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/util/swap" "k8s.io/klog/v2" @@ -32,8 +33,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + resourcehelper "k8s.io/component-helpers/resource" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm" usernamespacefeature "k8s.io/kubernetes/pkg/kubelet/userns" "k8s.io/kubernetes/pkg/volume" @@ -83,7 +84,7 @@ func (plugin *emptyDirPlugin) GetPluginName() string { func (plugin *emptyDirPlugin) GetVolumeName(spec *volume.Spec) (string, error) { volumeSource, _ := getVolumeSource(spec) if volumeSource == nil { - return "", fmt.Errorf("Spec does not reference an EmptyDir volume type") + return "", fmt.Errorf("spec does not reference an emptyDir volume type") } // Return user defined volume name, since this is an ephemeral volume type @@ -405,10 +406,19 @@ func getPageSizeMountOption(medium v1.StorageMedium, pod *v1.Pod) (string, error } } + podLevelAndContainerLevelRequests := []v1.ResourceList{} + if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) { + podLevelAndContainerLevelRequests = append(podLevelAndContainerLevelRequests, pod.Spec.Resources.Requests) + } + // In some rare cases init containers can also consume Huge pages for _, container := range append(pod.Spec.Containers, pod.Spec.InitContainers...) { - // We can take request because limit and requests must match. - for requestName := range container.Resources.Requests { + podLevelAndContainerLevelRequests = append(podLevelAndContainerLevelRequests, container.Resources.Requests) + } + + // We can take request because limit and requests must match. + for _, resourceList := range podLevelAndContainerLevelRequests { + for requestName := range resourceList { if !v1helper.IsHugePageResourceName(requestName) { continue } @@ -438,7 +448,6 @@ func getPageSizeMountOption(medium v1.StorageMedium, pod *v1.Pod) (string, error } return fmt.Sprintf("%s=%s", hugePagesPageSizeMountOption, pageSize.String()), nil - } // setupDir creates the directory with the default permissions specified by the perm constant. From 51db93c3fb474c3017eac49834d4c5a4ab247dbe Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 19 Feb 2025 19:54:13 +0000 Subject: [PATCH 4/8] Use pod level hugepage limits for cgroup when unset in container --- .../kuberuntime_container_linux.go | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go index de32e864a52..4fafdeda6f3 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go @@ -143,7 +143,7 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerResources(pod *v1.Pod, lcr.OomScoreAdj = int64(qos.GetContainerOOMScoreAdjust(pod, container, int64(m.machineInfo.MemoryCapacity))) - lcr.HugepageLimits = GetHugepageLimitsFromResources(container.Resources) + lcr.HugepageLimits = GetHugepageLimitsFromResources(pod, container.Resources) // Configure swap for the container m.configureContainerSwapResources(lcr, pod, container) @@ -300,7 +300,7 @@ func (m *kubeGenericRuntimeManager) calculateLinuxResources(cpuRequest, cpuLimit } // GetHugepageLimitsFromResources returns limits of each hugepages from resources. -func GetHugepageLimitsFromResources(resources v1.ResourceRequirements) []*runtimeapi.HugepageLimit { +func GetHugepageLimitsFromResources(pod *v1.Pod, resources v1.ResourceRequirements) []*runtimeapi.HugepageLimit { var hugepageLimits []*runtimeapi.HugepageLimit // For each page size, limit to 0. @@ -312,23 +312,20 @@ func GetHugepageLimitsFromResources(resources v1.ResourceRequirements) []*runtim } requiredHugepageLimits := map[string]uint64{} + + // When hugepage limits are specified at pod level, the container limit will + // match the pod level limit, however if the container also specifies, + // its limit will have precedence over the pod level limit + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) { + for limitName, limitAmount := range pod.Spec.Resources.Limits { + readAndDefineRequiredHugepageLimit(requiredHugepageLimits, limitName, limitAmount) + } + } + + // If the container and the pod specify, the container will have precedence, + // overriding the pod level limit for resourceObj, amountObj := range resources.Limits { - if !v1helper.IsHugePageResourceName(resourceObj) { - continue - } - - pageSize, err := v1helper.HugePageSizeFromResourceName(resourceObj) - if err != nil { - klog.InfoS("Failed to get hugepage size from resource", "object", resourceObj, "err", err) - continue - } - - sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value()) - if err != nil { - klog.InfoS("Size is invalid", "object", resourceObj, "err", err) - continue - } - requiredHugepageLimits[sizeString] = uint64(amountObj.Value()) + readAndDefineRequiredHugepageLimit(requiredHugepageLimits, resourceObj, amountObj) } for _, hugepageLimit := range hugepageLimits { @@ -340,6 +337,25 @@ func GetHugepageLimitsFromResources(resources v1.ResourceRequirements) []*runtim return hugepageLimits } +func readAndDefineRequiredHugepageLimit(requiredHugepageLimits map[string]uint64, resourceObj v1.ResourceName, amountObj resource.Quantity) { + if !v1helper.IsHugePageResourceName(resourceObj) { + return + } + + pageSize, err := v1helper.HugePageSizeFromResourceName(resourceObj) + if err != nil { + klog.InfoS("Failed to get hugepage size from resource", "object", resourceObj, "err", err) + return + } + + sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value()) + if err != nil { + klog.InfoS("Size is invalid", "object", resourceObj, "err", err) + return + } + requiredHugepageLimits[sizeString] = uint64(amountObj.Value()) +} + func toKubeContainerResources(statusResources *runtimeapi.ContainerResources) *kubecontainer.ContainerResources { var cStatusResources *kubecontainer.ContainerResources runtimeStatusResources := statusResources.GetLinux() From b9e0d4ad6667d0882ba57b34ca97ae77ef2585c0 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 26 Feb 2025 19:38:32 +0000 Subject: [PATCH 5/8] Unit tests for pod level hugepage resources --- pkg/apis/core/v1/defaults_test.go | 292 ++++++++++++++++++ pkg/apis/core/validation/validation_test.go | 173 +++++++---- .../kuberuntime_container_linux_test.go | 2 +- .../plugins/noderesources/fit_test.go | 60 +++- pkg/volume/emptydir/empty_dir_test.go | 126 +++++++- .../resource/helpers_test.go | 153 +++++++++ 6 files changed, 745 insertions(+), 61 deletions(-) diff --git a/pkg/apis/core/v1/defaults_test.go b/pkg/apis/core/v1/defaults_test.go index c5e3a9bb518..f09c450eb4f 100644 --- a/pkg/apis/core/v1/defaults_test.go +++ b/pkg/apis/core/v1/defaults_test.go @@ -1222,6 +1222,298 @@ func TestPodResourcesDefaults(t *testing.T) { }, }, }, + }, { + name: "pod hugepages requests=unset limits=unset, container hugepages requests=unset limits=set", + podLevelResourcesEnabled: true, + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + expectedPodSpec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("3m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("6Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("6Mi"), + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + }, + }, { + name: "pod hugepages requests=unset limits=set, container hugepages requests=unset limits=set", + podLevelResourcesEnabled: true, + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + expectedPodSpec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("3m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + }, + }, { + name: "pod hugepages requests=set limits=set, container hugepages requests=unset limits=set", + podLevelResourcesEnabled: true, + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + Requests: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + expectedPodSpec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + }, + }, { + name: "pod hugepages requests=set limits=set, container hugepages requests=unset limits=unset", + podLevelResourcesEnabled: true, + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + Requests: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{}, + }, + }, + expectedPodSpec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{}, + }, + }, + }, + }, { + name: "pod hugepages requests=unset limits=set, container hugepages requests=unset limits=set different hugepagesizes between pod and container level", + podLevelResourcesEnabled: true, + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + expectedPodSpec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("3m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + }, }} for _, tc := range cases { diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 23b4f417f5e..f009b0fd86f 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -19024,7 +19024,7 @@ func TestValidatePodResourceConsistency(t *testing.T) { }, }, }, { - name: "indivdual container limits greater than pod limits", + name: "individual container limits greater than pod limits", podResources: core.ResourceRequirements{ Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("10"), @@ -19084,6 +19084,8 @@ func TestValidatePodResourceNames(t *testing.T) { {"memory", false}, {"cpu", false}, {"storage", true}, + {v1.ResourceHugePagesPrefix + "2Mi", false}, + {v1.ResourceHugePagesPrefix + "1Gi", false}, {"requests.cpu", true}, {"requests.memory", true}, {"requests.storage", true}, @@ -19128,6 +19130,8 @@ func TestValidateResourceNames(t *testing.T) { {"memory", true, ""}, {"cpu", true, ""}, {"storage", true, ""}, + {v1.ResourceHugePagesPrefix + "2Mi", true, ""}, + {v1.ResourceHugePagesPrefix + "1Gi", true, ""}, {"requests.cpu", true, ""}, {"requests.memory", true, ""}, {"requests.storage", true, ""}, @@ -23624,6 +23628,48 @@ func TestValidateResourceRequirements(t *testing.T) { }, }, validateFn: ValidateContainerResourceRequirements, + }, { + name: "container resource hugepage with cpu or memory", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + core.ResourceCPU: resource.MustParse("10"), + }, + Requests: core.ResourceList{ + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + }, + validateFn: ValidateContainerResourceRequirements, + }, { + name: "container resource hugepage limit without request", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceMemory: resource.MustParse("2Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + }, + validateFn: ValidateContainerResourceRequirements, + }, { + name: "pod resource hugepages with cpu or memory", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + Requests: core.ResourceList{ + core.ResourceMemory: resource.MustParse("2Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + }, + validateFn: validatePodResourceRequirements, + }, { + name: "pod resource hugepages limit without request", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceMemory: resource.MustParse("2Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + }, + validateFn: validatePodResourceRequirements, }, { name: "limits and requests of memory resource are equal", requirements: core.ResourceRequirements{ @@ -23686,62 +23732,81 @@ func TestValidateResourceRequirements(t *testing.T) { validateFn func(requirements *core.ResourceRequirements, podClaimNames sets.Set[string], fldPath *field.Path, opts PodValidationOptions) field.ErrorList - }{{ - name: "hugepage resource without cpu or memory", - requirements: core.ResourceRequirements{ - Limits: core.ResourceList{ - core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }{ + { + name: "container resource hugepage without cpu or memory", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + Requests: core.ResourceList{ + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, }, - Requests: core.ResourceList{ - core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + validateFn: ValidateContainerResourceRequirements, + }, { + name: "container resource hugepage without limit", + requirements: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceMemory: resource.MustParse("2Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, }, + validateFn: ValidateContainerResourceRequirements, + }, { + name: "pod resource hugepages without cpu or memory", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + Requests: core.ResourceList{ + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + }, + validateFn: validatePodResourceRequirements, + }, { + name: "pod resource hugepages request without limit", + requirements: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceMemory: resource.MustParse("2Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), + }, + }, + validateFn: validatePodResourceRequirements, + }, { + name: "pod resource with ephemeral-storage", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceName(core.ResourceEphemeralStorage): resource.MustParse("2Mi"), + }, + Requests: core.ResourceList{ + core.ResourceName(core.ResourceEphemeralStorage + "2Mi"): resource.MustParse("2Mi"), + }, + }, + validateFn: validatePodResourceRequirements, + }, { + name: "pod resource with unsupported prefixed resources", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceName("kubernetesio/" + core.ResourceCPU): resource.MustParse("2"), + }, + Requests: core.ResourceList{ + core.ResourceName("kubernetesio/" + core.ResourceMemory): resource.MustParse("2"), + }, + }, + validateFn: validatePodResourceRequirements, + }, { + name: "pod resource with unsupported native resources", + requirements: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceName("kubernetes.io/" + strings.Repeat("a", 63)): resource.MustParse("2"), + }, + Requests: core.ResourceList{ + core.ResourceName("kubernetes.io/" + strings.Repeat("a", 63)): resource.MustParse("2"), + }, + }, + validateFn: validatePodResourceRequirements, }, - validateFn: ValidateContainerResourceRequirements, - }, { - name: "pod resource with hugepages", - requirements: core.ResourceRequirements{ - Limits: core.ResourceList{ - core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), - }, - Requests: core.ResourceList{ - core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), - }, - }, - validateFn: validatePodResourceRequirements, - }, { - name: "pod resource with ephemeral-storage", - requirements: core.ResourceRequirements{ - Limits: core.ResourceList{ - core.ResourceName(core.ResourceEphemeralStorage): resource.MustParse("2Mi"), - }, - Requests: core.ResourceList{ - core.ResourceName(core.ResourceEphemeralStorage + "2Mi"): resource.MustParse("2Mi"), - }, - }, - validateFn: validatePodResourceRequirements, - }, { - name: "pod resource with unsupported prefixed resources", - requirements: core.ResourceRequirements{ - Limits: core.ResourceList{ - core.ResourceName("kubernetesio/" + core.ResourceCPU): resource.MustParse("2"), - }, - Requests: core.ResourceList{ - core.ResourceName("kubernetesio/" + core.ResourceMemory): resource.MustParse("2"), - }, - }, - validateFn: validatePodResourceRequirements, - }, { - name: "pod resource with unsupported native resources", - requirements: core.ResourceRequirements{ - Limits: core.ResourceList{ - core.ResourceName("kubernetes.io/" + strings.Repeat("a", 63)): resource.MustParse("2"), - }, - Requests: core.ResourceList{ - core.ResourceName("kubernetes.io/" + strings.Repeat("a", 63)): resource.MustParse("2"), - }, - }, - validateFn: validatePodResourceRequirements, - }, { name: "pod resource with unsupported empty native resource name", requirements: core.ResourceRequirements{ diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go index 6bb6f606966..ae1ad6125d6 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go @@ -665,7 +665,7 @@ func TestGetHugepageLimitsFromResources(t *testing.T) { } } - results := GetHugepageLimitsFromResources(test.resources) + results := GetHugepageLimitsFromResources(&v1.Pod{}, test.resources) if !reflect.DeepEqual(expectedHugepages, results) { t.Errorf("%s test failed. Expected %v but got %v", test.name, expectedHugepages, results) } diff --git a/pkg/scheduler/framework/plugins/noderesources/fit_test.go b/pkg/scheduler/framework/plugins/noderesources/fit_test.go index 84727423ca4..ab46344d906 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit_test.go @@ -543,6 +543,45 @@ func TestEnoughRequests(t *testing.T) { ResourceName: v1.ResourceMemory, Reason: getErrReason(v1.ResourceMemory), Requested: 2, Used: 19, Capacity: 20}, }, }, + { + podLevelResourcesEnabled: true, + pod: newPodLevelResourcesPod( + newResourcePod(), + v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3m"), v1.ResourceMemory: resource.MustParse("2"), hugePageResourceA: *resource.NewQuantity(5, resource.BinarySI)}, + }, + ), + nodeInfo: framework.NewNodeInfo(), + name: "pod-level hugepages resource fit", + wantInsufficientResources: []InsufficientResource{}, + }, + { + podLevelResourcesEnabled: true, + pod: newPodLevelResourcesPod( + newResourcePod(framework.Resource{MilliCPU: 1, Memory: 1, ScalarResources: map[v1.ResourceName]int64{hugePageResourceA: 3}}), + v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3m"), v1.ResourceMemory: resource.MustParse("2"), hugePageResourceA: *resource.NewQuantity(5, resource.BinarySI)}, + }, + ), + nodeInfo: framework.NewNodeInfo(), + name: "both pod-level and container-level hugepages resource fit", + wantInsufficientResources: []InsufficientResource{}, + }, + { + podLevelResourcesEnabled: true, + pod: newPodLevelResourcesPod( + newResourcePod(), + v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3m"), v1.ResourceMemory: resource.MustParse("2"), hugePageResourceA: *resource.NewQuantity(10, resource.BinarySI)}, + }, + ), + nodeInfo: framework.NewNodeInfo(), + name: "pod-level hugepages resource not fit", + wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(hugePageResourceA)), + wantInsufficientResources: []InsufficientResource{ + {ResourceName: hugePageResourceA, Reason: getErrReason(hugePageResourceA), Requested: 10, Used: 0, Capacity: 5}, + }, + }, { podLevelResourcesEnabled: true, pod: newResourceInitPod(newPodLevelResourcesPod( @@ -1547,8 +1586,25 @@ func TestIsFit(t *testing.T) { pod: st.MakePod().Resources( v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}}, ).Obj(), - node: st.MakeNode().Capacity(map[v1.ResourceName]string{v1.ResourceCPU: "2"}).Obj(), - expected: true, + node: st.MakeNode().Capacity(map[v1.ResourceName]string{v1.ResourceCPU: "2"}).Obj(), + podLevelResourcesEnabled: true, + expected: true, + }, + "sufficient pod-level resource hugepages": { + pod: st.MakePod().Resources( + v1.ResourceRequirements{Requests: v1.ResourceList{hugePageResourceA: resource.MustParse("2Mi")}}, + ).Obj(), + node: st.MakeNode().Capacity(map[v1.ResourceName]string{hugePageResourceA: "2Mi"}).Obj(), + podLevelResourcesEnabled: true, + expected: true, + }, + "insufficient pod-level resource hugepages": { + pod: st.MakePod().Resources( + v1.ResourceRequirements{Requests: v1.ResourceList{hugePageResourceA: resource.MustParse("4Mi")}}, + ).Obj(), + node: st.MakeNode().Capacity(map[v1.ResourceName]string{hugePageResourceA: "2Mi"}).Obj(), + podLevelResourcesEnabled: true, + expected: false, }, } diff --git a/pkg/volume/emptydir/empty_dir_test.go b/pkg/volume/emptydir/empty_dir_test.go index 5dcf9c0f7b5..73826145871 100644 --- a/pkg/volume/emptydir/empty_dir_test.go +++ b/pkg/volume/emptydir/empty_dir_test.go @@ -26,6 +26,9 @@ import ( "strings" "testing" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/util/swap" v1 "k8s.io/api/core/v1" @@ -373,10 +376,11 @@ func TestMetrics(t *testing.T) { func TestGetHugePagesMountOptions(t *testing.T) { testCases := map[string]struct { - pod *v1.Pod - medium v1.StorageMedium - shouldFail bool - expectedResult string + pod *v1.Pod + medium v1.StorageMedium + shouldFail bool + expectedResult string + podLevelResourcesEnabled bool }{ "ProperValues": { pod: &v1.Pod{ @@ -605,10 +609,124 @@ func TestGetHugePagesMountOptions(t *testing.T) { shouldFail: true, expectedResult: "", }, + "PodLevelResourcesSinglePageSize": { + podLevelResourcesEnabled: true, + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), + }, + }, + }, + }, + medium: v1.StorageMediumHugePages, + shouldFail: false, + expectedResult: "pagesize=2Mi", + }, + "PodLevelResourcesSinglePageSizeMediumPrefix": { + podLevelResourcesEnabled: true, + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), + }, + }, + }, + }, + medium: v1.StorageMediumHugePagesPrefix + "2Mi", + shouldFail: false, + expectedResult: "pagesize=2Mi", + }, + "PodLevelResourcesMultiPageSize": { + podLevelResourcesEnabled: true, + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("hugepages-1Gi"): resource.MustParse("2Gi"), + v1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), + }, + }, + }, + }, + medium: v1.StorageMediumHugePages, + shouldFail: true, + expectedResult: "", + }, + "PodLevelResourcesMultiPageSizeMediumPrefix": { + podLevelResourcesEnabled: true, + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("hugepages-1Gi"): resource.MustParse("2Gi"), + v1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), + }, + }, + }, + }, + medium: v1.StorageMediumHugePagesPrefix + "2Mi", + shouldFail: false, + expectedResult: "pagesize=2Mi", + }, + "PodAndContainerLevelResourcesMultiPageSizeHugePagesMedium": { + podLevelResourcesEnabled: true, + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("hugepages-1Gi"): resource.MustParse("2Gi"), + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), + }, + }, + }, + }, + }, + }, + medium: v1.StorageMediumHugePages, + shouldFail: true, + expectedResult: "", + }, + "PodAndContainerLevelResourcesMultiPageSizeHugePagesMediumPrefix": { + podLevelResourcesEnabled: true, + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("hugepages-1Gi"): resource.MustParse("2Gi"), + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("hugepages-2Mi"): resource.MustParse("100Mi"), + }, + }, + }, + }, + }, + }, + medium: v1.StorageMediumHugePagesPrefix + "2Mi", + shouldFail: false, + expectedResult: "pagesize=2Mi", + }, } for testCaseName, testCase := range testCases { t.Run(testCaseName, func(t *testing.T) { + if testCase.podLevelResourcesEnabled { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, true) + } + value, err := getPageSizeMountOption(testCase.medium, testCase.pod) if testCase.shouldFail && err == nil { t.Errorf("%s: Unexpected success", testCaseName) diff --git a/staging/src/k8s.io/component-helpers/resource/helpers_test.go b/staging/src/k8s.io/component-helpers/resource/helpers_test.go index a5d4657d21f..24e0ca03676 100644 --- a/staging/src/k8s.io/component-helpers/resource/helpers_test.go +++ b/staging/src/k8s.io/component-helpers/resource/helpers_test.go @@ -1710,6 +1710,118 @@ func TestPodLevelResourceRequests(t *testing.T) { opts: PodResourcesOptions{SkipPodLevelResources: false}, expectedRequests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("15Mi"), v1.ResourceCPU: resource.MustParse("18m")}, }, + { + name: "pod-level resources, hugepage request/limit single page size", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("10Mi"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + opts: PodResourcesOptions{SkipPodLevelResources: false}, + expectedRequests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("10Mi"), v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi")}, + }, + { + name: "pod-level resources, hugepage request/limit multiple page sizes", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + }, + opts: PodResourcesOptions{SkipPodLevelResources: false}, + expectedRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi")}, + }, + { + name: "pod-level resources, container-level resources, hugepage request/limit single page size", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("6Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("6Mi"), + }, + }, + }, + }, + opts: PodResourcesOptions{SkipPodLevelResources: false}, + expectedRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi")}, + }, + { + name: "pod-level resources, container-level resources, hugepage request/limit multiple page sizes", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("2Gi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("2Gi"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("2Gi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("2Gi"), + }, + }, + }, + }, + opts: PodResourcesOptions{SkipPodLevelResources: false}, + expectedRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("2Gi")}, + }, + { + name: "pod-level resources, container-level resources, hugepage request/limit multiple page sizes between pod-level and container-level", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("4Mi"), + v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi"), + }, + }, + }, + }, + opts: PodResourcesOptions{SkipPodLevelResources: false}, + expectedRequests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("4Mi"), v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("10Mi"), v1.ResourceHugePagesPrefix + "1Gi": resource.MustParse("1Gi")}, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -1721,6 +1833,47 @@ func TestPodLevelResourceRequests(t *testing.T) { } } +func TestIsSupportedPodLevelResource(t *testing.T) { + testCases := []struct { + name string + resource v1.ResourceName + expected bool + }{ + { + name: v1.ResourceCPU.String(), + resource: v1.ResourceCPU, + expected: true, + }, + { + name: v1.ResourceMemory.String(), + resource: v1.ResourceMemory, + expected: true, + }, + { + name: v1.ResourceEphemeralStorage.String(), + resource: v1.ResourceEphemeralStorage, + expected: false, + }, + { + name: v1.ResourceHugePagesPrefix + "2Mi", + resource: v1.ResourceHugePagesPrefix + "2Mi", + expected: true, + }, + { + name: v1.ResourceHugePagesPrefix + "1Gi", + resource: v1.ResourceHugePagesPrefix + "1Gi", + expected: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if got := IsSupportedPodLevelResource(tc.resource); got != tc.expected { + t.Errorf("Supported pod level resource %s: got=%t, want=%t", tc.resource.String(), got, tc.expected) + } + }) + } +} + func TestAggregateContainerRequestsAndLimits(t *testing.T) { restartAlways := v1.ContainerRestartPolicyAlways cases := []struct { From fd50d9207f51f9fa665ac6bbdf97a8db1c0a34e8 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Fri, 28 Feb 2025 23:01:36 +0000 Subject: [PATCH 6/8] End to end tests for pod level hugepage resources --- test/e2e/common/node/pod_level_resources.go | 253 +++++++++++++++++--- test/e2e_node/hugepages_test.go | 132 +--------- test/e2e_node/memory_manager_test.go | 7 +- test/utils/node.go | 189 ++++++++++++++- 4 files changed, 411 insertions(+), 170 deletions(-) diff --git a/test/e2e/common/node/pod_level_resources.go b/test/e2e/common/node/pod_level_resources.go index cf5c1f5b5e7..3dd58ff051c 100644 --- a/test/e2e/common/node/pod_level_resources.go +++ b/test/e2e/common/node/pod_level_resources.go @@ -29,12 +29,17 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" + v1resource "k8s.io/kubernetes/pkg/api/v1/resource" + v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + + utils "k8s.io/kubernetes/test/utils" + imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" ) @@ -43,9 +48,14 @@ const ( cgroupv2CPUWeight string = "cpu.weight" cgroupv2CPULimit string = "cpu.max" cgroupv2MemLimit string = "memory.max" - cgroupFsPath string = "/sys/fs/cgroup" - CPUPeriod string = "100000" - mountPath string = "/sysfscgroup" + + cgroupv2HugeTLBPrefix string = "hugetlb" + cgroupv2HugeTLBRsvd string = "rsvd" + + cgroupFsPath string = "/sys/fs/cgroup" + mountPath string = "/sysfscgroup" + + CPUPeriod string = "100000" ) var ( @@ -69,6 +79,7 @@ var _ = SIGDescribe("Pod Level Resources", framework.WithSerial(), feature.PodLe e2eskipper.Skipf("not supported on cgroupv1 -- skipping") } }) + podLevelResourcesTests(f) }) @@ -104,7 +115,7 @@ func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool { func makeObjectMetadata(name, namespace string) metav1.ObjectMeta { return metav1.ObjectMeta{ - Name: "testpod", Namespace: namespace, + Name: name, Namespace: namespace, Labels: map[string]string{"time": strconv.Itoa(time.Now().Nanosecond())}, } } @@ -113,11 +124,16 @@ type containerInfo struct { Name string Resources *resourceInfo } + type resourceInfo struct { - CPUReq string - CPULim string - MemReq string - MemLim string + CPUReq string + CPULim string + MemReq string + MemLim string + HugePagesReq2Mi string + HugePagesLim2Mi string + HugePagesReq1Gi string + HugePagesLim1Gi string } func makeContainer(info containerInfo) v1.Container { @@ -140,7 +156,7 @@ func makeContainer(info containerInfo) v1.Container { func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements { var res v1.ResourceRequirements if info != nil { - if info.CPUReq != "" || info.MemReq != "" { + if info.CPUReq != "" || info.MemReq != "" || info.HugePagesReq2Mi != "" || info.HugePagesReq1Gi != "" { res.Requests = make(v1.ResourceList) } if info.CPUReq != "" { @@ -149,8 +165,14 @@ func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements { if info.MemReq != "" { res.Requests[v1.ResourceMemory] = resource.MustParse(info.MemReq) } + if info.HugePagesReq2Mi != "" { + res.Requests[v1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse(info.HugePagesReq2Mi) + } + if info.HugePagesReq1Gi != "" { + res.Requests[v1.ResourceHugePagesPrefix+"1Gi"] = resource.MustParse(info.HugePagesReq1Gi) + } - if info.CPULim != "" || info.MemLim != "" { + if info.CPULim != "" || info.MemLim != "" || info.HugePagesLim2Mi != "" || info.HugePagesLim1Gi != "" { res.Limits = make(v1.ResourceList) } if info.CPULim != "" { @@ -159,6 +181,12 @@ func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements { if info.MemLim != "" { res.Limits[v1.ResourceMemory] = resource.MustParse(info.MemLim) } + if info.HugePagesLim2Mi != "" { + res.Limits[v1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse(info.HugePagesLim2Mi) + } + if info.HugePagesLim1Gi != "" { + res.Limits[v1.ResourceHugePagesPrefix+"1Gi"] = resource.MustParse(info.HugePagesLim1Gi) + } } return res } @@ -211,7 +239,7 @@ func verifyQoS(gotPod v1.Pod, expectedQoS v1.PodQOSClass) { } // TODO(ndixita): dedup the conversion logic in pod resize test and move to helpers/utils. -func verifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, info *resourceInfo) error { +func verifyPodCgroups(f *framework.Framework, pod *v1.Pod, info *resourceInfo) error { ginkgo.GinkgoHelper() cmd := fmt.Sprintf("find %s -name '*%s*'", mountPath, strings.ReplaceAll(string(pod.UID), "-", "_")) framework.Logf("Namespace %s Pod %s - looking for Pod cgroup directory path: %q", f.Namespace, pod.Name, cmd) @@ -247,6 +275,70 @@ func verifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, if err != nil { errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) } + + // Verify cgroup limits for all the hugepage sizes in the pod + for resourceName, resourceAmount := range expectedResources.Limits { + if !v1resource.IsHugePageResourceName(resourceName) { + continue + } + + pageSize, err := v1helper.HugePageSizeFromResourceName(resourceName) + if err != nil { + errs = append(errs, fmt.Errorf("encountered error while obtaining hugepage size: %w", err)) + } + + sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value()) + if err != nil { + errs = append(errs, fmt.Errorf("encountered error while obtaining hugepage unit size: %w", err)) + } + + hugepageCgroupv2Limits := []string{ + fmt.Sprintf("%s.%s.max", cgroupv2HugeTLBPrefix, sizeString), + fmt.Sprintf("%s.%s.%s.max", cgroupv2HugeTLBPrefix, sizeString, cgroupv2HugeTLBRsvd), + } + expectedHugepageLim := strconv.FormatInt(resourceAmount.Value(), 10) + + for _, hugepageCgroupv2Limit := range hugepageCgroupv2Limits { + hugepageLimCgPath := fmt.Sprintf("%s/%s", podCgPath, hugepageCgroupv2Limit) + err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, hugepageLimCgPath, expectedHugepageLim) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify hugepage limit cgroup value: %w, path: %s", err, hugepageLimCgPath)) + } + } + } + + return utilerrors.NewAggregate(errs) +} + +func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error { + var errs []error + for _, container := range pod.Spec.Containers { + if pod.Spec.Resources == nil { + continue + } + + if pod.Spec.Resources.Limits.Memory() != nil && container.Resources.Limits.Memory() == nil { + expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10) + err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) + } + } + + if pod.Spec.Resources.Limits.Cpu() != nil && container.Resources.Limits.Cpu() == nil { + cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod) + expectedCPULimit := strconv.FormatInt(cpuQuota, 10) + expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod) + err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)) + } + } + + // TODO(KevinTMtz) - Check for all hugepages for the pod, for this is + // required to enabled the Containerd Cgroup value, because if not, HugeTLB + // cgroup values will be just set to max + } return utilerrors.NewAggregate(errs) } @@ -257,7 +349,7 @@ func podLevelResourcesTests(f *framework.Framework) { // and limits for the pod. If pod-level resource specifications // are specified, totalPodResources is equal to pod-level resources. // Otherwise, it is calculated by aggregating resource requests and - // limits from all containers within the pod.. + // limits from all containers within the pod. totalPodResources *resourceInfo } @@ -266,6 +358,7 @@ func podLevelResourcesTests(f *framework.Framework) { podResources *resourceInfo containers []containerInfo expected expectedPodConfig + hugepages map[string]int } tests := []testCase{ @@ -349,10 +442,108 @@ func podLevelResourcesTests(f *framework.Framework) { totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, }, }, + { + name: "Guaranteed QoS pod hugepages, no container resources, single page size", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"}, + containers: []containerInfo{{Name: "c1"}, {Name: "c2"}}, + expected: expectedPodConfig{ + qos: v1.PodQOSGuaranteed, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi"}, + }, + hugepages: map[string]int{ + v1.ResourceHugePagesPrefix + "2Mi": 5, + }, + }, + { + name: "Burstable QoS pod hugepages, container resources, single page size", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"}, + containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi"}}, {Name: "c2"}}, + expected: expectedPodConfig{ + qos: v1.PodQOSBurstable, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi"}, + }, + hugepages: map[string]int{ + v1.ResourceHugePagesPrefix + "2Mi": 5, + }, + }, + { + name: "Burstable QoS pod hugepages, container resources, single page size, pod level does not specify hugepages", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, + containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi"}}, {Name: "c2"}}, + expected: expectedPodConfig{ + qos: v1.PodQOSBurstable, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "4Mi", HugePagesLim2Mi: "4Mi"}, + }, + hugepages: map[string]int{ + v1.ResourceHugePagesPrefix + "2Mi": 2, + }, + }, + { + name: "Guaranteed QoS pod hugepages, no container resources, multiple page size", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi", HugePagesLim1Gi: "1Gi"}, + containers: []containerInfo{{Name: "c1"}, {Name: "c2"}}, + expected: expectedPodConfig{ + qos: v1.PodQOSGuaranteed, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"}, + }, + hugepages: map[string]int{ + v1.ResourceHugePagesPrefix + "2Mi": 5, + v1.ResourceHugePagesPrefix + "1Gi": 1, + }, + }, + { + name: "Burstable QoS pod hugepages, container resources, multiple page size", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi", HugePagesLim1Gi: "1Gi"}, + containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}}, + expected: expectedPodConfig{ + qos: v1.PodQOSBurstable, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"}, + }, + hugepages: map[string]int{ + v1.ResourceHugePagesPrefix + "2Mi": 5, + v1.ResourceHugePagesPrefix + "1Gi": 1, + }, + }, + { + name: "Burstable QoS pod hugepages, container resources, multiple page size, pod level does not specify hugepages", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, + containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}}, + expected: expectedPodConfig{ + qos: v1.PodQOSBurstable, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "4Mi", HugePagesLim2Mi: "4Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"}, + }, + hugepages: map[string]int{ + v1.ResourceHugePagesPrefix + "2Mi": 2, + v1.ResourceHugePagesPrefix + "1Gi": 1, + }, + }, + { + name: "Burstable QoS pod hugepages, container resources, different page size between pod and container level", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"}, + containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}}, + expected: expectedPodConfig{ + qos: v1.PodQOSBurstable, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"}, + }, + hugepages: map[string]int{ + v1.ResourceHugePagesPrefix + "2Mi": 5, + v1.ResourceHugePagesPrefix + "1Gi": 1, + }, + }, } for _, tc := range tests { ginkgo.It(tc.name, func(ctx context.Context) { + // Pre-allocate hugepages in the node + if tc.hugepages != nil { + utils.SetHugepages(ctx, tc.hugepages) + + ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") + utils.RestartKubelet(ctx, false) + + utils.WaitForHugepages(ctx, f, tc.hugepages) + } + podMetadata := makeObjectMetadata("testpod", f.Namespace.Name) testPod := makePod(&podMetadata, tc.podResources, tc.containers) @@ -367,7 +558,7 @@ func podLevelResourcesTests(f *framework.Framework) { verifyQoS(*pod, tc.expected.qos) ginkgo.By("verifying pod cgroup values") - err := verifyPodCgroups(ctx, f, pod, tc.expected.totalPodResources) + err := verifyPodCgroups(f, pod, tc.expected.totalPodResources) framework.ExpectNoError(err, "failed to verify pod's cgroup values: %v", err) ginkgo.By("verifying containers cgroup limits are same as pod container's cgroup limits") @@ -377,32 +568,16 @@ func podLevelResourcesTests(f *framework.Framework) { ginkgo.By("deleting pods") delErr := e2epod.DeletePodWithWait(ctx, f.ClientSet, pod) framework.ExpectNoError(delErr, "failed to delete pod %s", delErr) + + // Release pre-allocated hugepages + if tc.hugepages != nil { + utils.ReleaseHugepages(ctx, tc.hugepages) + + ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") + utils.RestartKubelet(ctx, true) + + utils.WaitForHugepages(ctx, f, tc.hugepages) + } }) } } - -func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error { - var errs []error - for _, container := range pod.Spec.Containers { - if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Memory() != nil && - container.Resources.Limits.Memory() == nil { - expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10) - err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit) - if err != nil { - errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) - } - } - - if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Cpu() != nil && - container.Resources.Limits.Cpu() == nil { - cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod) - expectedCPULimit := strconv.FormatInt(cpuQuota, 10) - expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod) - err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit) - if err != nil { - errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)) - } - } - } - return utilerrors.NewAggregate(errs) -} diff --git a/test/e2e_node/hugepages_test.go b/test/e2e_node/hugepages_test.go index 4330f026cd1..bc2bd6474ed 100644 --- a/test/e2e_node/hugepages_test.go +++ b/test/e2e_node/hugepages_test.go @@ -19,10 +19,6 @@ package e2enode import ( "context" "fmt" - "os" - "os/exec" - "strconv" - "strings" "time" "github.com/onsi/ginkgo/v2" @@ -37,7 +33,7 @@ import ( "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + testutils "k8s.io/kubernetes/test/utils" admissionapi "k8s.io/pod-security-admission/api" ) @@ -119,64 +115,6 @@ func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity, return pod } -// configureHugePages attempts to allocate hugepages of the specified size -func configureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error { - // Compact memory to make bigger contiguous blocks of memory available - // before allocating huge pages. - // https://www.kernel.org/doc/Documentation/sysctl/vm.txt - if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil { - if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil { - return err - } - } - - // e.g. hugepages/hugepages-2048kB/nr_hugepages - hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile) - - // e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix) - if numaNodeID != nil { - // e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages - hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix) - } - - // Reserve number of hugepages - // e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" - command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile) - if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { - return err - } - - // verify that the number of hugepages was updated - // e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages" - command = fmt.Sprintf("cat %s", hugepagesFile) - outData, err := exec.Command("/bin/sh", "-c", command).Output() - if err != nil { - return err - } - - numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData))) - if err != nil { - return err - } - - framework.Logf("Hugepages total is set to %v", numHugePages) - if numHugePages == hugepagesCount { - return nil - } - - return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages) -} - -// isHugePageAvailable returns true if hugepages of the specified size is available on the host -func isHugePageAvailable(hugepagesSize int) bool { - path := fmt.Sprintf("%s-%dkB/%s", hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile) - if _, err := os.Stat(path); err != nil { - return false - } - return true -} - func getHugepagesTestPod(f *framework.Framework, limits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -262,66 +200,6 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func hugepages map[string]int ) - setHugepages := func(ctx context.Context) { - for hugepagesResource, count := range hugepages { - size := resourceToSize[hugepagesResource] - ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size)) - if !isHugePageAvailable(size) { - e2eskipper.Skipf("skipping test because hugepages of size %d not supported", size) - return - } - - ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size)) - gomega.Eventually(ctx, func() error { - if err := configureHugePages(size, count, nil); err != nil { - return err - } - return nil - }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) - } - } - - waitForHugepages := func(ctx context.Context) { - ginkgo.By("Waiting for hugepages resource to become available on the local node") - gomega.Eventually(ctx, func(ctx context.Context) error { - node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{}) - if err != nil { - return err - } - - for hugepagesResource, count := range hugepages { - capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)] - if !ok { - return fmt.Errorf("the node does not have the resource %s", hugepagesResource) - } - - size, succeed := capacity.AsInt64() - if !succeed { - return fmt.Errorf("failed to convert quantity to int64") - } - - expectedSize := count * resourceToSize[hugepagesResource] * 1024 - if size != int64(expectedSize) { - return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize) - } - } - return nil - }, time.Minute, framework.Poll).Should(gomega.BeNil()) - } - - releaseHugepages := func(ctx context.Context) { - ginkgo.By("Releasing hugepages") - gomega.Eventually(ctx, func() error { - for hugepagesResource := range hugepages { - command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile) - if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { - return err - } - } - return nil - }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) - } - runHugePagesTests := func() { ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { ginkgo.By("getting mounts for the test pod") @@ -349,12 +227,12 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func // setup ginkgo.JustBeforeEach(func(ctx context.Context) { - setHugepages(ctx) + testutils.SetHugepages(ctx, hugepages) ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") restartKubelet(ctx, true) - waitForHugepages(ctx) + testutils.WaitForHugepages(ctx, f, hugepages) pod := getHugepagesTestPod(f, limits, mounts, volumes) @@ -367,12 +245,12 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func ginkgo.By(fmt.Sprintf("deleting test pod %s", testpod.Name)) e2epod.NewPodClient(f).DeleteSync(ctx, testpod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete) - releaseHugepages(ctx) + testutils.ReleaseHugepages(ctx, hugepages) ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") restartKubelet(ctx, true) - waitForHugepages(ctx) + testutils.WaitForHugepages(ctx, f, hugepages) }) ginkgo.Context("with the resources requests that contain only one hugepages resource ", func() { diff --git a/test/e2e_node/memory_manager_test.go b/test/e2e_node/memory_manager_test.go index 1ff4688d59c..15764bb6925 100644 --- a/test/e2e_node/memory_manager_test.go +++ b/test/e2e_node/memory_manager_test.go @@ -42,6 +42,7 @@ import ( "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + testutils "k8s.io/kubernetes/test/utils" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/utils/cpuset" "k8s.io/utils/pointer" @@ -314,7 +315,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With } if is2MiHugepagesSupported == nil { - is2MiHugepagesSupported = pointer.BoolPtr(isHugePageAvailable(hugepagesSize2M)) + is2MiHugepagesSupported = pointer.BoolPtr(testutils.IsHugePageAvailable(hugepagesSize2M)) } if len(allNUMANodes) == 0 { @@ -325,7 +326,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With if *is2MiHugepagesSupported { ginkgo.By("Configuring hugepages") gomega.Eventually(ctx, func() error { - return configureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0)) + return testutils.ConfigureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0)) }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) } }) @@ -358,7 +359,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With ginkgo.By("Releasing allocated hugepages") gomega.Eventually(ctx, func() error { // configure hugepages on the NUMA node 0 to avoid hugepages split across NUMA nodes - return configureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0)) + return testutils.ConfigureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0)) }, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages") } }) diff --git a/test/utils/node.go b/test/utils/node.go index 3483088aa72..bcd4bab3c72 100644 --- a/test/utils/node.go +++ b/test/utils/node.go @@ -16,7 +16,39 @@ limitations under the License. package utils -import v1 "k8s.io/api/core/v1" +import ( + "context" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + hugepagesCapacityFile = "nr_hugepages" + hugepagesDirPrefix = "/sys/kernel/mm/hugepages/hugepages" + + hugepagesSize2M = 2048 + hugepagesSize1G = 1048576 +) + +var ( + resourceToSize = map[string]int{ + v1.ResourceHugePagesPrefix + "2Mi": hugepagesSize2M, + v1.ResourceHugePagesPrefix + "1Gi": hugepagesSize1G, + } +) // GetNodeCondition extracts the provided condition from the given status and returns that. // Returns nil and -1 if the condition is not present, and the index of the located condition. @@ -31,3 +63,158 @@ func GetNodeCondition(status *v1.NodeStatus, conditionType v1.NodeConditionType) } return -1, nil } + +func SetHugepages(ctx context.Context, hugepages map[string]int) { + for hugepagesResource, count := range hugepages { + size := resourceToSize[hugepagesResource] + ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size)) + if !IsHugePageAvailable(size) { + skipf("skipping test because hugepages of size %d not supported", size) + return + } + + ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size)) + gomega.Eventually(ctx, func() error { + if err := ConfigureHugePages(size, count, nil); err != nil { + return err + } + return nil + }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) + } +} + +func IsHugePageAvailable(size int) bool { + // e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + hugepagesFile := fmt.Sprintf("/sys/kernel/mm/hugepages/hugepages-%dkB/nr_hugepages", size) + if _, err := os.Stat(hugepagesFile); err != nil { + framework.Logf("Hugepages file %s not found: %v", hugepagesFile, err) + return false + } + return true +} + +// configureHugePages attempts to allocate hugepages of the specified size +func ConfigureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error { + // Compact memory to make bigger contiguous blocks of memory available + // before allocating huge pages. + // https://www.kernel.org/doc/Documentation/sysctl/vm.txt + if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil { + if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil { + return err + } + } + + // e.g. hugepages/hugepages-2048kB/nr_hugepages + hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile) + + // e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix) + if numaNodeID != nil { + // e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages + hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix) + } + + // Reserve number of hugepages + // e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" + command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile) + if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { + return err + } + + // verify that the number of hugepages was updated + // e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages" + command = fmt.Sprintf("cat %s", hugepagesFile) + outData, err := exec.Command("/bin/sh", "-c", command).Output() + if err != nil { + return err + } + + numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData))) + if err != nil { + return err + } + + framework.Logf("Hugepages total is set to %v", numHugePages) + if numHugePages == hugepagesCount { + return nil + } + + return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages) +} + +// TODO(KevinTMtz) - Deduplicate from test/e2e_node/util.go:restartKubelet +func RestartKubelet(ctx context.Context, running bool) { + kubeletServiceName := FindKubeletServiceName(running) + // reset the kubelet service start-limit-hit + stdout, err := exec.CommandContext(ctx, "sudo", "systemctl", "reset-failed", kubeletServiceName).CombinedOutput() + framework.ExpectNoError(err, "Failed to reset kubelet start-limit-hit with systemctl: %v, %s", err, string(stdout)) + + stdout, err = exec.CommandContext(ctx, "sudo", "systemctl", "restart", kubeletServiceName).CombinedOutput() + framework.ExpectNoError(err, "Failed to restart kubelet with systemctl: %v, %s", err, string(stdout)) +} + +func FindKubeletServiceName(running bool) string { + cmdLine := []string{ + "systemctl", "list-units", "*kubelet*", + } + if running { + cmdLine = append(cmdLine, "--state=running") + } + stdout, err := exec.Command("sudo", cmdLine...).CombinedOutput() + framework.ExpectNoError(err) + regex := regexp.MustCompile("(kubelet-\\w+)") + matches := regex.FindStringSubmatch(string(stdout)) + gomega.Expect(matches).ToNot(gomega.BeEmpty(), "Found more than one kubelet service running: %q", stdout) + kubeletServiceName := matches[0] + framework.Logf("Get running kubelet with systemctl: %v, %v", string(stdout), kubeletServiceName) + return kubeletServiceName +} + +func WaitForHugepages(ctx context.Context, f *framework.Framework, hugepages map[string]int) { + ginkgo.By("Waiting for hugepages resource to become available on the local node") + gomega.Eventually(ctx, func(ctx context.Context) error { + node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + for hugepagesResource, count := range hugepages { + capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)] + if !ok { + return fmt.Errorf("the node does not have the resource %s", hugepagesResource) + } + + size, succeed := capacity.AsInt64() + if !succeed { + return fmt.Errorf("failed to convert quantity to int64") + } + + expectedSize := count * resourceToSize[hugepagesResource] * 1024 + if size != int64(expectedSize) { + return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize) + } + } + return nil + }, time.Minute, framework.Poll).Should(gomega.BeNil()) +} + +func ReleaseHugepages(ctx context.Context, hugepages map[string]int) { + ginkgo.By("Releasing hugepages") + gomega.Eventually(ctx, func() error { + for hugepagesResource := range hugepages { + command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile) + if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { + return err + } + } + return nil + }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) +} + +// TODO(KevinTMtz) - Deduplicate from test/e2e/framework/skipper/skipper.go:Skipf +func skipf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + ginkgo.Skip(msg, 2) + + panic("unreachable") +} From 98e362c04952438d8a934f20bec0a19ce1473119 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Fri, 7 Mar 2025 00:12:31 +0000 Subject: [PATCH 7/8] Revert "Pod level hugepage for cgroup when unset in container" This reverts commit f0a14732371cb6f9521567b3fad564d8a8360bd7 and kuberuntime_container_linux_test.go change. --- .../kuberuntime_container_linux.go | 52 +++++++------------ .../kuberuntime_container_linux_test.go | 2 +- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go index 4fafdeda6f3..de32e864a52 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go @@ -143,7 +143,7 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerResources(pod *v1.Pod, lcr.OomScoreAdj = int64(qos.GetContainerOOMScoreAdjust(pod, container, int64(m.machineInfo.MemoryCapacity))) - lcr.HugepageLimits = GetHugepageLimitsFromResources(pod, container.Resources) + lcr.HugepageLimits = GetHugepageLimitsFromResources(container.Resources) // Configure swap for the container m.configureContainerSwapResources(lcr, pod, container) @@ -300,7 +300,7 @@ func (m *kubeGenericRuntimeManager) calculateLinuxResources(cpuRequest, cpuLimit } // GetHugepageLimitsFromResources returns limits of each hugepages from resources. -func GetHugepageLimitsFromResources(pod *v1.Pod, resources v1.ResourceRequirements) []*runtimeapi.HugepageLimit { +func GetHugepageLimitsFromResources(resources v1.ResourceRequirements) []*runtimeapi.HugepageLimit { var hugepageLimits []*runtimeapi.HugepageLimit // For each page size, limit to 0. @@ -312,20 +312,23 @@ func GetHugepageLimitsFromResources(pod *v1.Pod, resources v1.ResourceRequiremen } requiredHugepageLimits := map[string]uint64{} - - // When hugepage limits are specified at pod level, the container limit will - // match the pod level limit, however if the container also specifies, - // its limit will have precedence over the pod level limit - if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) { - for limitName, limitAmount := range pod.Spec.Resources.Limits { - readAndDefineRequiredHugepageLimit(requiredHugepageLimits, limitName, limitAmount) - } - } - - // If the container and the pod specify, the container will have precedence, - // overriding the pod level limit for resourceObj, amountObj := range resources.Limits { - readAndDefineRequiredHugepageLimit(requiredHugepageLimits, resourceObj, amountObj) + if !v1helper.IsHugePageResourceName(resourceObj) { + continue + } + + pageSize, err := v1helper.HugePageSizeFromResourceName(resourceObj) + if err != nil { + klog.InfoS("Failed to get hugepage size from resource", "object", resourceObj, "err", err) + continue + } + + sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value()) + if err != nil { + klog.InfoS("Size is invalid", "object", resourceObj, "err", err) + continue + } + requiredHugepageLimits[sizeString] = uint64(amountObj.Value()) } for _, hugepageLimit := range hugepageLimits { @@ -337,25 +340,6 @@ func GetHugepageLimitsFromResources(pod *v1.Pod, resources v1.ResourceRequiremen return hugepageLimits } -func readAndDefineRequiredHugepageLimit(requiredHugepageLimits map[string]uint64, resourceObj v1.ResourceName, amountObj resource.Quantity) { - if !v1helper.IsHugePageResourceName(resourceObj) { - return - } - - pageSize, err := v1helper.HugePageSizeFromResourceName(resourceObj) - if err != nil { - klog.InfoS("Failed to get hugepage size from resource", "object", resourceObj, "err", err) - return - } - - sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value()) - if err != nil { - klog.InfoS("Size is invalid", "object", resourceObj, "err", err) - return - } - requiredHugepageLimits[sizeString] = uint64(amountObj.Value()) -} - func toKubeContainerResources(statusResources *runtimeapi.ContainerResources) *kubecontainer.ContainerResources { var cStatusResources *kubecontainer.ContainerResources runtimeStatusResources := statusResources.GetLinux() diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go index ae1ad6125d6..6bb6f606966 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go @@ -665,7 +665,7 @@ func TestGetHugepageLimitsFromResources(t *testing.T) { } } - results := GetHugepageLimitsFromResources(&v1.Pod{}, test.resources) + results := GetHugepageLimitsFromResources(test.resources) if !reflect.DeepEqual(expectedHugepages, results) { t.Errorf("%s test failed. Expected %v but got %v", test.name, expectedHugepages, results) } From 7b38bff6ec98b3c21fe8000423300beaafd6f73e Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Sun, 9 Mar 2025 05:09:58 +0000 Subject: [PATCH 8/8] Move pod level hugepage test from e2e to e2e node --- test/e2e/common/node/pod_level_resources.go | 253 ++------ test/e2e_node/hugepages_test.go | 679 ++++++++++++++++++-- test/e2e_node/memory_manager_test.go | 7 +- test/utils/node.go | 189 +----- 4 files changed, 673 insertions(+), 455 deletions(-) diff --git a/test/e2e/common/node/pod_level_resources.go b/test/e2e/common/node/pod_level_resources.go index 3dd58ff051c..cf5c1f5b5e7 100644 --- a/test/e2e/common/node/pod_level_resources.go +++ b/test/e2e/common/node/pod_level_resources.go @@ -29,17 +29,12 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" - v1resource "k8s.io/kubernetes/pkg/api/v1/resource" - v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - - utils "k8s.io/kubernetes/test/utils" - imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" ) @@ -48,14 +43,9 @@ const ( cgroupv2CPUWeight string = "cpu.weight" cgroupv2CPULimit string = "cpu.max" cgroupv2MemLimit string = "memory.max" - - cgroupv2HugeTLBPrefix string = "hugetlb" - cgroupv2HugeTLBRsvd string = "rsvd" - - cgroupFsPath string = "/sys/fs/cgroup" - mountPath string = "/sysfscgroup" - - CPUPeriod string = "100000" + cgroupFsPath string = "/sys/fs/cgroup" + CPUPeriod string = "100000" + mountPath string = "/sysfscgroup" ) var ( @@ -79,7 +69,6 @@ var _ = SIGDescribe("Pod Level Resources", framework.WithSerial(), feature.PodLe e2eskipper.Skipf("not supported on cgroupv1 -- skipping") } }) - podLevelResourcesTests(f) }) @@ -115,7 +104,7 @@ func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool { func makeObjectMetadata(name, namespace string) metav1.ObjectMeta { return metav1.ObjectMeta{ - Name: name, Namespace: namespace, + Name: "testpod", Namespace: namespace, Labels: map[string]string{"time": strconv.Itoa(time.Now().Nanosecond())}, } } @@ -124,16 +113,11 @@ type containerInfo struct { Name string Resources *resourceInfo } - type resourceInfo struct { - CPUReq string - CPULim string - MemReq string - MemLim string - HugePagesReq2Mi string - HugePagesLim2Mi string - HugePagesReq1Gi string - HugePagesLim1Gi string + CPUReq string + CPULim string + MemReq string + MemLim string } func makeContainer(info containerInfo) v1.Container { @@ -156,7 +140,7 @@ func makeContainer(info containerInfo) v1.Container { func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements { var res v1.ResourceRequirements if info != nil { - if info.CPUReq != "" || info.MemReq != "" || info.HugePagesReq2Mi != "" || info.HugePagesReq1Gi != "" { + if info.CPUReq != "" || info.MemReq != "" { res.Requests = make(v1.ResourceList) } if info.CPUReq != "" { @@ -165,14 +149,8 @@ func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements { if info.MemReq != "" { res.Requests[v1.ResourceMemory] = resource.MustParse(info.MemReq) } - if info.HugePagesReq2Mi != "" { - res.Requests[v1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse(info.HugePagesReq2Mi) - } - if info.HugePagesReq1Gi != "" { - res.Requests[v1.ResourceHugePagesPrefix+"1Gi"] = resource.MustParse(info.HugePagesReq1Gi) - } - if info.CPULim != "" || info.MemLim != "" || info.HugePagesLim2Mi != "" || info.HugePagesLim1Gi != "" { + if info.CPULim != "" || info.MemLim != "" { res.Limits = make(v1.ResourceList) } if info.CPULim != "" { @@ -181,12 +159,6 @@ func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements { if info.MemLim != "" { res.Limits[v1.ResourceMemory] = resource.MustParse(info.MemLim) } - if info.HugePagesLim2Mi != "" { - res.Limits[v1.ResourceHugePagesPrefix+"2Mi"] = resource.MustParse(info.HugePagesLim2Mi) - } - if info.HugePagesLim1Gi != "" { - res.Limits[v1.ResourceHugePagesPrefix+"1Gi"] = resource.MustParse(info.HugePagesLim1Gi) - } } return res } @@ -239,7 +211,7 @@ func verifyQoS(gotPod v1.Pod, expectedQoS v1.PodQOSClass) { } // TODO(ndixita): dedup the conversion logic in pod resize test and move to helpers/utils. -func verifyPodCgroups(f *framework.Framework, pod *v1.Pod, info *resourceInfo) error { +func verifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, info *resourceInfo) error { ginkgo.GinkgoHelper() cmd := fmt.Sprintf("find %s -name '*%s*'", mountPath, strings.ReplaceAll(string(pod.UID), "-", "_")) framework.Logf("Namespace %s Pod %s - looking for Pod cgroup directory path: %q", f.Namespace, pod.Name, cmd) @@ -275,70 +247,6 @@ func verifyPodCgroups(f *framework.Framework, pod *v1.Pod, info *resourceInfo) e if err != nil { errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) } - - // Verify cgroup limits for all the hugepage sizes in the pod - for resourceName, resourceAmount := range expectedResources.Limits { - if !v1resource.IsHugePageResourceName(resourceName) { - continue - } - - pageSize, err := v1helper.HugePageSizeFromResourceName(resourceName) - if err != nil { - errs = append(errs, fmt.Errorf("encountered error while obtaining hugepage size: %w", err)) - } - - sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value()) - if err != nil { - errs = append(errs, fmt.Errorf("encountered error while obtaining hugepage unit size: %w", err)) - } - - hugepageCgroupv2Limits := []string{ - fmt.Sprintf("%s.%s.max", cgroupv2HugeTLBPrefix, sizeString), - fmt.Sprintf("%s.%s.%s.max", cgroupv2HugeTLBPrefix, sizeString, cgroupv2HugeTLBRsvd), - } - expectedHugepageLim := strconv.FormatInt(resourceAmount.Value(), 10) - - for _, hugepageCgroupv2Limit := range hugepageCgroupv2Limits { - hugepageLimCgPath := fmt.Sprintf("%s/%s", podCgPath, hugepageCgroupv2Limit) - err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, hugepageLimCgPath, expectedHugepageLim) - if err != nil { - errs = append(errs, fmt.Errorf("failed to verify hugepage limit cgroup value: %w, path: %s", err, hugepageLimCgPath)) - } - } - } - - return utilerrors.NewAggregate(errs) -} - -func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error { - var errs []error - for _, container := range pod.Spec.Containers { - if pod.Spec.Resources == nil { - continue - } - - if pod.Spec.Resources.Limits.Memory() != nil && container.Resources.Limits.Memory() == nil { - expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10) - err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit) - if err != nil { - errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) - } - } - - if pod.Spec.Resources.Limits.Cpu() != nil && container.Resources.Limits.Cpu() == nil { - cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod) - expectedCPULimit := strconv.FormatInt(cpuQuota, 10) - expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod) - err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit) - if err != nil { - errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)) - } - } - - // TODO(KevinTMtz) - Check for all hugepages for the pod, for this is - // required to enabled the Containerd Cgroup value, because if not, HugeTLB - // cgroup values will be just set to max - } return utilerrors.NewAggregate(errs) } @@ -349,7 +257,7 @@ func podLevelResourcesTests(f *framework.Framework) { // and limits for the pod. If pod-level resource specifications // are specified, totalPodResources is equal to pod-level resources. // Otherwise, it is calculated by aggregating resource requests and - // limits from all containers within the pod. + // limits from all containers within the pod.. totalPodResources *resourceInfo } @@ -358,7 +266,6 @@ func podLevelResourcesTests(f *framework.Framework) { podResources *resourceInfo containers []containerInfo expected expectedPodConfig - hugepages map[string]int } tests := []testCase{ @@ -442,108 +349,10 @@ func podLevelResourcesTests(f *framework.Framework) { totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, }, }, - { - name: "Guaranteed QoS pod hugepages, no container resources, single page size", - podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"}, - containers: []containerInfo{{Name: "c1"}, {Name: "c2"}}, - expected: expectedPodConfig{ - qos: v1.PodQOSGuaranteed, - totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi"}, - }, - hugepages: map[string]int{ - v1.ResourceHugePagesPrefix + "2Mi": 5, - }, - }, - { - name: "Burstable QoS pod hugepages, container resources, single page size", - podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"}, - containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi"}}, {Name: "c2"}}, - expected: expectedPodConfig{ - qos: v1.PodQOSBurstable, - totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi"}, - }, - hugepages: map[string]int{ - v1.ResourceHugePagesPrefix + "2Mi": 5, - }, - }, - { - name: "Burstable QoS pod hugepages, container resources, single page size, pod level does not specify hugepages", - podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, - containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi"}}, {Name: "c2"}}, - expected: expectedPodConfig{ - qos: v1.PodQOSBurstable, - totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "4Mi", HugePagesLim2Mi: "4Mi"}, - }, - hugepages: map[string]int{ - v1.ResourceHugePagesPrefix + "2Mi": 2, - }, - }, - { - name: "Guaranteed QoS pod hugepages, no container resources, multiple page size", - podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi", HugePagesLim1Gi: "1Gi"}, - containers: []containerInfo{{Name: "c1"}, {Name: "c2"}}, - expected: expectedPodConfig{ - qos: v1.PodQOSGuaranteed, - totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"}, - }, - hugepages: map[string]int{ - v1.ResourceHugePagesPrefix + "2Mi": 5, - v1.ResourceHugePagesPrefix + "1Gi": 1, - }, - }, - { - name: "Burstable QoS pod hugepages, container resources, multiple page size", - podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi", HugePagesLim1Gi: "1Gi"}, - containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}}, - expected: expectedPodConfig{ - qos: v1.PodQOSBurstable, - totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"}, - }, - hugepages: map[string]int{ - v1.ResourceHugePagesPrefix + "2Mi": 5, - v1.ResourceHugePagesPrefix + "1Gi": 1, - }, - }, - { - name: "Burstable QoS pod hugepages, container resources, multiple page size, pod level does not specify hugepages", - podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, - containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim2Mi: "4Mi", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}}, - expected: expectedPodConfig{ - qos: v1.PodQOSBurstable, - totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "4Mi", HugePagesLim2Mi: "4Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"}, - }, - hugepages: map[string]int{ - v1.ResourceHugePagesPrefix + "2Mi": 2, - v1.ResourceHugePagesPrefix + "1Gi": 1, - }, - }, - { - name: "Burstable QoS pod hugepages, container resources, different page size between pod and container level", - podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesLim2Mi: "10Mi"}, - containers: []containerInfo{{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "50m", HugePagesLim1Gi: "1Gi"}}, {Name: "c2"}}, - expected: expectedPodConfig{ - qos: v1.PodQOSBurstable, - totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi", HugePagesReq2Mi: "10Mi", HugePagesLim2Mi: "10Mi", HugePagesReq1Gi: "1Gi", HugePagesLim1Gi: "1Gi"}, - }, - hugepages: map[string]int{ - v1.ResourceHugePagesPrefix + "2Mi": 5, - v1.ResourceHugePagesPrefix + "1Gi": 1, - }, - }, } for _, tc := range tests { ginkgo.It(tc.name, func(ctx context.Context) { - // Pre-allocate hugepages in the node - if tc.hugepages != nil { - utils.SetHugepages(ctx, tc.hugepages) - - ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") - utils.RestartKubelet(ctx, false) - - utils.WaitForHugepages(ctx, f, tc.hugepages) - } - podMetadata := makeObjectMetadata("testpod", f.Namespace.Name) testPod := makePod(&podMetadata, tc.podResources, tc.containers) @@ -558,7 +367,7 @@ func podLevelResourcesTests(f *framework.Framework) { verifyQoS(*pod, tc.expected.qos) ginkgo.By("verifying pod cgroup values") - err := verifyPodCgroups(f, pod, tc.expected.totalPodResources) + err := verifyPodCgroups(ctx, f, pod, tc.expected.totalPodResources) framework.ExpectNoError(err, "failed to verify pod's cgroup values: %v", err) ginkgo.By("verifying containers cgroup limits are same as pod container's cgroup limits") @@ -568,16 +377,32 @@ func podLevelResourcesTests(f *framework.Framework) { ginkgo.By("deleting pods") delErr := e2epod.DeletePodWithWait(ctx, f.ClientSet, pod) framework.ExpectNoError(delErr, "failed to delete pod %s", delErr) - - // Release pre-allocated hugepages - if tc.hugepages != nil { - utils.ReleaseHugepages(ctx, tc.hugepages) - - ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") - utils.RestartKubelet(ctx, true) - - utils.WaitForHugepages(ctx, f, tc.hugepages) - } }) } } + +func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error { + var errs []error + for _, container := range pod.Spec.Containers { + if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Memory() != nil && + container.Resources.Limits.Memory() == nil { + expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10) + err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) + } + } + + if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Cpu() != nil && + container.Resources.Limits.Cpu() == nil { + cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod) + expectedCPULimit := strconv.FormatInt(cpuQuota, 10) + expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod) + err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)) + } + } + } + return utilerrors.NewAggregate(errs) +} diff --git a/test/e2e_node/hugepages_test.go b/test/e2e_node/hugepages_test.go index bc2bd6474ed..12900faeed2 100644 --- a/test/e2e_node/hugepages_test.go +++ b/test/e2e_node/hugepages_test.go @@ -19,6 +19,10 @@ package e2enode import ( "context" "fmt" + "os" + "os/exec" + "strconv" + "strings" "time" "github.com/onsi/ginkgo/v2" @@ -29,11 +33,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - testutils "k8s.io/kubernetes/test/utils" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" admissionapi "k8s.io/pod-security-admission/api" ) @@ -115,8 +120,66 @@ func makePodToVerifyHugePages(baseName string, hugePagesLimit resource.Quantity, return pod } -func getHugepagesTestPod(f *framework.Framework, limits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { - return &v1.Pod{ +// configureHugePages attempts to allocate hugepages of the specified size +func configureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error { + // Compact memory to make bigger contiguous blocks of memory available + // before allocating huge pages. + // https://www.kernel.org/doc/Documentation/sysctl/vm.txt + if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil { + if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil { + return err + } + } + + // e.g. hugepages/hugepages-2048kB/nr_hugepages + hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile) + + // e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix) + if numaNodeID != nil { + // e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages + hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix) + } + + // Reserve number of hugepages + // e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" + command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile) + if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { + return err + } + + // verify that the number of hugepages was updated + // e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages" + command = fmt.Sprintf("cat %s", hugepagesFile) + outData, err := exec.Command("/bin/sh", "-c", command).Output() + if err != nil { + return err + } + + numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData))) + if err != nil { + return err + } + + framework.Logf("Hugepages total is set to %v", numHugePages) + if numHugePages == hugepagesCount { + return nil + } + + return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages) +} + +// isHugePageAvailable returns true if hugepages of the specified size is available on the host +func isHugePageAvailable(hugepagesSize int) bool { + path := fmt.Sprintf("%s-%dkB/%s", hugepagesDirPrefix, hugepagesSize, hugepagesCapacityFile) + if _, err := os.Stat(path); err != nil { + return false + } + return true +} + +func getHugepagesTestPod(f *framework.Framework, podLimits v1.ResourceList, containerLimits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "hugepages-", Namespace: f.Namespace.Name, @@ -124,18 +187,110 @@ func getHugepagesTestPod(f *framework.Framework, limits v1.ResourceList, mounts Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "container" + string(uuid.NewUUID()), - Image: busyboxImage, - Resources: v1.ResourceRequirements{ - Limits: limits, - }, + Name: "container" + string(uuid.NewUUID()), + Image: busyboxImage, Command: []string{"sleep", "3600"}, VolumeMounts: mounts, + Resources: v1.ResourceRequirements{ + Limits: containerLimits, + }, }, }, Volumes: volumes, }, } + + if podLimits != nil { + pod.Spec.Resources = &v1.ResourceRequirements{ + Limits: podLimits, + } + } + + return pod +} + +func setHugepages(ctx context.Context, hugepages map[string]int) { + for hugepagesResource, count := range hugepages { + size := resourceToSize[hugepagesResource] + ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size)) + if !isHugePageAvailable(size) { + e2eskipper.Skipf("skipping test because hugepages of size %d not supported", size) + return + } + + ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size)) + gomega.Eventually(ctx, func() error { + if err := configureHugePages(size, count, nil); err != nil { + return err + } + return nil + }, 30*time.Second, framework.Poll).Should(gomega.Succeed()) + } +} + +func waitForHugepages(f *framework.Framework, ctx context.Context, hugepages map[string]int) { + ginkgo.By("Waiting for hugepages resource to become available on the local node") + gomega.Eventually(ctx, func(ctx context.Context) error { + node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + for hugepagesResource, count := range hugepages { + capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)] + if !ok { + return fmt.Errorf("the node does not have the resource %s", hugepagesResource) + } + + size, succeed := capacity.AsInt64() + if !succeed { + return fmt.Errorf("failed to convert quantity to int64") + } + + expectedSize := count * resourceToSize[hugepagesResource] * 1024 + if size != int64(expectedSize) { + return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize) + } + } + return nil + }, time.Minute, framework.Poll).Should(gomega.Succeed()) +} + +func releaseHugepages(ctx context.Context, hugepages map[string]int) { + ginkgo.By("Releasing hugepages") + gomega.Eventually(ctx, func() error { + for hugepagesResource := range hugepages { + command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile) + if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { + return err + } + } + return nil + }, 30*time.Second, framework.Poll).Should(gomega.Succeed()) +} + +func runHugePagesTests(f *framework.Framework, ctx context.Context, testpod *v1.Pod, expectedHugepageLimits v1.ResourceList, mounts []v1.VolumeMount, hugepages map[string]int) { + ginkgo.By("getting mounts for the test pod") + command := []string{"mount"} + + out := e2epod.ExecCommandInContainer(f, testpod.Name, testpod.Spec.Containers[0].Name, command...) + + for _, mount := range mounts { + ginkgo.By(fmt.Sprintf("checking that the hugetlb mount %s exists under the container", mount.MountPath)) + gomega.Expect(out).To(gomega.ContainSubstring(mount.MountPath)) + } + + for resourceName := range hugepages { + verifyPod := makePodToVerifyHugePages( + "pod"+string(testpod.UID), + expectedHugepageLimits[v1.ResourceName(resourceName)], + resourceToCgroup[resourceName], + ) + ginkgo.By("checking if the expected hugetlb settings were applied") + e2epod.NewPodClient(f).Create(ctx, verifyPod) + err := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, verifyPod.Name, f.Namespace.Name) + framework.ExpectNoError(err) + } } // Serial because the test updates kubelet configuration. @@ -193,48 +348,24 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func ginkgo.When("start the pod", func() { var ( - testpod *v1.Pod - limits v1.ResourceList - mounts []v1.VolumeMount - volumes []v1.Volume - hugepages map[string]int + testpod *v1.Pod + expectedHugepageLimits v1.ResourceList + containerLimits v1.ResourceList + mounts []v1.VolumeMount + volumes []v1.Volume + hugepages map[string]int ) - runHugePagesTests := func() { - ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { - ginkgo.By("getting mounts for the test pod") - command := []string{"mount"} - out := e2epod.ExecCommandInContainer(f, testpod.Name, testpod.Spec.Containers[0].Name, command...) - - for _, mount := range mounts { - ginkgo.By(fmt.Sprintf("checking that the hugetlb mount %s exists under the container", mount.MountPath)) - gomega.Expect(out).To(gomega.ContainSubstring(mount.MountPath)) - } - - for resourceName := range hugepages { - verifyPod := makePodToVerifyHugePages( - "pod"+string(testpod.UID), - testpod.Spec.Containers[0].Resources.Limits[v1.ResourceName(resourceName)], - resourceToCgroup[resourceName], - ) - ginkgo.By("checking if the expected hugetlb settings were applied") - e2epod.NewPodClient(f).Create(ctx, verifyPod) - err := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, verifyPod.Name, f.Namespace.Name) - framework.ExpectNoError(err) - } - }) - } - // setup ginkgo.JustBeforeEach(func(ctx context.Context) { - testutils.SetHugepages(ctx, hugepages) + setHugepages(ctx, hugepages) ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") restartKubelet(ctx, true) - testutils.WaitForHugepages(ctx, f, hugepages) + waitForHugepages(f, ctx, hugepages) - pod := getHugepagesTestPod(f, limits, mounts, volumes) + pod := getHugepagesTestPod(f, nil, containerLimits, mounts, volumes) ginkgo.By("by running a test pod that requests hugepages") testpod = e2epod.NewPodClient(f).CreateSync(ctx, pod) @@ -245,18 +376,21 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func ginkgo.By(fmt.Sprintf("deleting test pod %s", testpod.Name)) e2epod.NewPodClient(f).DeleteSync(ctx, testpod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete) - testutils.ReleaseHugepages(ctx, hugepages) + releaseHugepages(ctx, hugepages) ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") restartKubelet(ctx, true) - testutils.WaitForHugepages(ctx, f, hugepages) + waitForHugepages(f, ctx, hugepages) }) ginkgo.Context("with the resources requests that contain only one hugepages resource ", func() { ginkgo.Context("with the backward compatible API", func() { ginkgo.BeforeEach(func() { - limits = v1.ResourceList{ + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), v1.ResourceMemory: resource.MustParse("100Mi"), hugepagesResourceName2Mi: resource.MustParse("6Mi"), @@ -280,12 +414,17 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func hugepages = map[string]int{hugepagesResourceName2Mi: 5} }) // run tests - runHugePagesTests() + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) }) ginkgo.Context("with the new API", func() { ginkgo.BeforeEach(func() { - limits = v1.ResourceList{ + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), v1.ResourceMemory: resource.MustParse("100Mi"), hugepagesResourceName2Mi: resource.MustParse("6Mi"), @@ -309,7 +448,9 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func hugepages = map[string]int{hugepagesResourceName2Mi: 5} }) - runHugePagesTests() + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) }) ginkgo.JustAfterEach(func() { @@ -323,7 +464,11 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func hugepagesResourceName2Mi: 5, hugepagesResourceName1Gi: 1, } - limits = v1.ResourceList{ + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), v1.ResourceMemory: resource.MustParse("100Mi"), hugepagesResourceName2Mi: resource.MustParse("6Mi"), @@ -359,7 +504,443 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func } }) - runHugePagesTests() + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + hugepagesResourceName1Gi: 0, + } + }) + }) + }) +}) + +// Serial because the test updates kubelet configuration. +var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), feature.PodLevelResources, func() { + f := framework.NewDefaultFramework("pod-level-hugepages-resources") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + ginkgo.When("pod level resources", func() { + var ( + testpod *v1.Pod + expectedHugepageLimits v1.ResourceList + podLimits v1.ResourceList + containerLimits v1.ResourceList + mounts []v1.VolumeMount + volumes []v1.Volume + hugepages map[string]int + ) + + // setup + ginkgo.JustBeforeEach(func(ctx context.Context) { + e2eskipper.SkipUnlessFeatureGateEnabled(features.PodLevelResources) + + setHugepages(ctx, hugepages) + + ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") + restartKubelet(ctx, true) + + waitForHugepages(f, ctx, hugepages) + + pod := getHugepagesTestPod(f, podLimits, containerLimits, mounts, volumes) + + ginkgo.By("by running a test pod that requests hugepages") + + testpod = e2epod.NewPodClient(f).CreateSync(ctx, pod) + + framework.Logf("Test pod name: %s", testpod.Name) + }) + + // we should use JustAfterEach because framework will teardown the client under the AfterEach method + ginkgo.JustAfterEach(func(ctx context.Context) { + ginkgo.By(fmt.Sprintf("deleting test pod %s", testpod.Name)) + e2epod.NewPodClient(f).DeleteSync(ctx, testpod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete) + + releaseHugepages(ctx, hugepages) + + ginkgo.By("restarting kubelet to pick up pre-allocated hugepages") + restartKubelet(ctx, true) + + waitForHugepages(f, ctx, hugepages) + }) + + ginkgo.Context("pod hugepages, no container hugepages, single page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + podLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + containerLimits = v1.ResourceList{} + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + } + }) + }) + + ginkgo.Context("pod hugepages, container hugepages, single page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + podLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + } + }) + }) + + ginkgo.Context("no pod hugepages, container hugepages, single page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + } + podLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + } + }) + }) + + ginkgo.Context("pod hugepages, no container hugepages, multiple page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + hugepagesResourceName1Gi: 1, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + podLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + containerLimits = v1.ResourceList{} + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + { + Name: "hugepages-1gi", + MountPath: "/hugepages-1Gi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + { + Name: "hugepages-1gi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages1Gi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + hugepagesResourceName1Gi: 0, + } + }) + }) + + ginkgo.Context("pod hugepages, container hugepages, multiple page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + hugepagesResourceName1Gi: 1, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + podLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + { + Name: "hugepages-1gi", + MountPath: "/hugepages-1Gi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + { + Name: "hugepages-1gi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages1Gi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + hugepagesResourceName1Gi: 0, + } + }) + }) + + ginkgo.Context("no pod hugepages, container hugepages, multiple page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + hugepagesResourceName1Gi: 1, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + podLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + { + Name: "hugepages-1gi", + MountPath: "/hugepages-1Gi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + { + Name: "hugepages-1gi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages1Gi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + hugepagesResourceName1Gi: 0, + } + }) + }) + + ginkgo.Context("pod hugepages, container hugepages, different page size between pod and container level", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + hugepagesResourceName1Gi: 1, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + podLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + { + Name: "hugepages-1gi", + MountPath: "/hugepages-1Gi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + { + Name: "hugepages-1gi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages1Gi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) ginkgo.JustAfterEach(func() { hugepages = map[string]int{ diff --git a/test/e2e_node/memory_manager_test.go b/test/e2e_node/memory_manager_test.go index 15764bb6925..1ff4688d59c 100644 --- a/test/e2e_node/memory_manager_test.go +++ b/test/e2e_node/memory_manager_test.go @@ -42,7 +42,6 @@ import ( "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - testutils "k8s.io/kubernetes/test/utils" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/utils/cpuset" "k8s.io/utils/pointer" @@ -315,7 +314,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With } if is2MiHugepagesSupported == nil { - is2MiHugepagesSupported = pointer.BoolPtr(testutils.IsHugePageAvailable(hugepagesSize2M)) + is2MiHugepagesSupported = pointer.BoolPtr(isHugePageAvailable(hugepagesSize2M)) } if len(allNUMANodes) == 0 { @@ -326,7 +325,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With if *is2MiHugepagesSupported { ginkgo.By("Configuring hugepages") gomega.Eventually(ctx, func() error { - return testutils.ConfigureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0)) + return configureHugePages(hugepagesSize2M, hugepages2MiCount, pointer.IntPtr(0)) }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) } }) @@ -359,7 +358,7 @@ var _ = SIGDescribe("Memory Manager", framework.WithDisruptive(), framework.With ginkgo.By("Releasing allocated hugepages") gomega.Eventually(ctx, func() error { // configure hugepages on the NUMA node 0 to avoid hugepages split across NUMA nodes - return testutils.ConfigureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0)) + return configureHugePages(hugepagesSize2M, 0, pointer.IntPtr(0)) }, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages") } }) diff --git a/test/utils/node.go b/test/utils/node.go index bcd4bab3c72..3483088aa72 100644 --- a/test/utils/node.go +++ b/test/utils/node.go @@ -16,39 +16,7 @@ limitations under the License. package utils -import ( - "context" - "fmt" - "os" - "os/exec" - "regexp" - "strconv" - "strings" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/kubernetes/test/e2e/framework" -) - -const ( - hugepagesCapacityFile = "nr_hugepages" - hugepagesDirPrefix = "/sys/kernel/mm/hugepages/hugepages" - - hugepagesSize2M = 2048 - hugepagesSize1G = 1048576 -) - -var ( - resourceToSize = map[string]int{ - v1.ResourceHugePagesPrefix + "2Mi": hugepagesSize2M, - v1.ResourceHugePagesPrefix + "1Gi": hugepagesSize1G, - } -) +import v1 "k8s.io/api/core/v1" // GetNodeCondition extracts the provided condition from the given status and returns that. // Returns nil and -1 if the condition is not present, and the index of the located condition. @@ -63,158 +31,3 @@ func GetNodeCondition(status *v1.NodeStatus, conditionType v1.NodeConditionType) } return -1, nil } - -func SetHugepages(ctx context.Context, hugepages map[string]int) { - for hugepagesResource, count := range hugepages { - size := resourceToSize[hugepagesResource] - ginkgo.By(fmt.Sprintf("Verifying hugepages %d are supported", size)) - if !IsHugePageAvailable(size) { - skipf("skipping test because hugepages of size %d not supported", size) - return - } - - ginkgo.By(fmt.Sprintf("Configuring the host to reserve %d of pre-allocated hugepages of size %d", count, size)) - gomega.Eventually(ctx, func() error { - if err := ConfigureHugePages(size, count, nil); err != nil { - return err - } - return nil - }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) - } -} - -func IsHugePageAvailable(size int) bool { - // e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - hugepagesFile := fmt.Sprintf("/sys/kernel/mm/hugepages/hugepages-%dkB/nr_hugepages", size) - if _, err := os.Stat(hugepagesFile); err != nil { - framework.Logf("Hugepages file %s not found: %v", hugepagesFile, err) - return false - } - return true -} - -// configureHugePages attempts to allocate hugepages of the specified size -func ConfigureHugePages(hugepagesSize int, hugepagesCount int, numaNodeID *int) error { - // Compact memory to make bigger contiguous blocks of memory available - // before allocating huge pages. - // https://www.kernel.org/doc/Documentation/sysctl/vm.txt - if _, err := os.Stat("/proc/sys/vm/compact_memory"); err == nil { - if err := exec.Command("/bin/sh", "-c", "echo 1 > /proc/sys/vm/compact_memory").Run(); err != nil { - return err - } - } - - // e.g. hugepages/hugepages-2048kB/nr_hugepages - hugepagesSuffix := fmt.Sprintf("hugepages/hugepages-%dkB/%s", hugepagesSize, hugepagesCapacityFile) - - // e.g. /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - hugepagesFile := fmt.Sprintf("/sys/kernel/mm/%s", hugepagesSuffix) - if numaNodeID != nil { - // e.g. /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages - hugepagesFile = fmt.Sprintf("/sys/devices/system/node/node%d/%s", *numaNodeID, hugepagesSuffix) - } - - // Reserve number of hugepages - // e.g. /bin/sh -c "echo 5 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" - command := fmt.Sprintf("echo %d > %s", hugepagesCount, hugepagesFile) - if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { - return err - } - - // verify that the number of hugepages was updated - // e.g. /bin/sh -c "cat /sys/kernel/mm/hugepages/hugepages-2048kB/vm.nr_hugepages" - command = fmt.Sprintf("cat %s", hugepagesFile) - outData, err := exec.Command("/bin/sh", "-c", command).Output() - if err != nil { - return err - } - - numHugePages, err := strconv.Atoi(strings.TrimSpace(string(outData))) - if err != nil { - return err - } - - framework.Logf("Hugepages total is set to %v", numHugePages) - if numHugePages == hugepagesCount { - return nil - } - - return fmt.Errorf("expected hugepages %v, but found %v", hugepagesCount, numHugePages) -} - -// TODO(KevinTMtz) - Deduplicate from test/e2e_node/util.go:restartKubelet -func RestartKubelet(ctx context.Context, running bool) { - kubeletServiceName := FindKubeletServiceName(running) - // reset the kubelet service start-limit-hit - stdout, err := exec.CommandContext(ctx, "sudo", "systemctl", "reset-failed", kubeletServiceName).CombinedOutput() - framework.ExpectNoError(err, "Failed to reset kubelet start-limit-hit with systemctl: %v, %s", err, string(stdout)) - - stdout, err = exec.CommandContext(ctx, "sudo", "systemctl", "restart", kubeletServiceName).CombinedOutput() - framework.ExpectNoError(err, "Failed to restart kubelet with systemctl: %v, %s", err, string(stdout)) -} - -func FindKubeletServiceName(running bool) string { - cmdLine := []string{ - "systemctl", "list-units", "*kubelet*", - } - if running { - cmdLine = append(cmdLine, "--state=running") - } - stdout, err := exec.Command("sudo", cmdLine...).CombinedOutput() - framework.ExpectNoError(err) - regex := regexp.MustCompile("(kubelet-\\w+)") - matches := regex.FindStringSubmatch(string(stdout)) - gomega.Expect(matches).ToNot(gomega.BeEmpty(), "Found more than one kubelet service running: %q", stdout) - kubeletServiceName := matches[0] - framework.Logf("Get running kubelet with systemctl: %v, %v", string(stdout), kubeletServiceName) - return kubeletServiceName -} - -func WaitForHugepages(ctx context.Context, f *framework.Framework, hugepages map[string]int) { - ginkgo.By("Waiting for hugepages resource to become available on the local node") - gomega.Eventually(ctx, func(ctx context.Context) error { - node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{}) - if err != nil { - return err - } - - for hugepagesResource, count := range hugepages { - capacity, ok := node.Status.Capacity[v1.ResourceName(hugepagesResource)] - if !ok { - return fmt.Errorf("the node does not have the resource %s", hugepagesResource) - } - - size, succeed := capacity.AsInt64() - if !succeed { - return fmt.Errorf("failed to convert quantity to int64") - } - - expectedSize := count * resourceToSize[hugepagesResource] * 1024 - if size != int64(expectedSize) { - return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize) - } - } - return nil - }, time.Minute, framework.Poll).Should(gomega.BeNil()) -} - -func ReleaseHugepages(ctx context.Context, hugepages map[string]int) { - ginkgo.By("Releasing hugepages") - gomega.Eventually(ctx, func() error { - for hugepagesResource := range hugepages { - command := fmt.Sprintf("echo 0 > %s-%dkB/%s", hugepagesDirPrefix, resourceToSize[hugepagesResource], hugepagesCapacityFile) - if err := exec.Command("/bin/sh", "-c", command).Run(); err != nil { - return err - } - } - return nil - }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) -} - -// TODO(KevinTMtz) - Deduplicate from test/e2e/framework/skipper/skipper.go:Skipf -func skipf(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - ginkgo.Skip(msg, 2) - - panic("unreachable") -}