Merge pull request #131487 from hshiina/dedup-cgroup-verification

e2e: Deduplicate cgroup verification

test/e2e/common/node/framework/cgroups/cgroups.go

@@ -0,0 +1,430 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cgroups
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	kubecm "k8s.io/kubernetes/pkg/kubelet/cm"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+)
+
+const (
+	cgroupFsPath          string = "/sys/fs/cgroup"
+	cgroupCPUSharesFile   string = "cpu.shares"
+	cgroupCPUQuotaFile    string = "cpu.cfs_quota_us"
+	cgroupMemLimitFile    string = "memory.limit_in_bytes"
+	cgroupv2CPUWeightFile string = "cpu.weight"
+	cgroupv2CPULimitFile  string = "cpu.max"
+	cgroupv2MemLimitFile  string = "memory.max"
+	cgroupVolumeName      string = "sysfscgroup"
+	cgroupMountPath       string = "/sysfscgroup"
+)
+
+var (
+	// TODO: cgroup version shouldn't be cached as a global for a cluster where v1 and v2 are mixed.
+	podOnCgroupv2Node      *bool
+	podOnCgroupv2NodeMutex sync.Mutex
+)
+
+type ContainerResources struct {
+	CPUReq              string
+	CPULim              string
+	MemReq              string
+	MemLim              string
+	EphStorReq          string
+	EphStorLim          string
+	ExtendedResourceReq string
+	ExtendedResourceLim string
+}
+
+func (cr *ContainerResources) ResourceRequirements() *v1.ResourceRequirements {
+	if cr == nil {
+		return nil
+	}
+
+	var lim, req v1.ResourceList
+	if cr.CPULim != "" || cr.MemLim != "" || cr.EphStorLim != "" {
+		lim = make(v1.ResourceList)
+	}
+	if cr.CPUReq != "" || cr.MemReq != "" || cr.EphStorReq != "" {
+		req = make(v1.ResourceList)
+	}
+	if cr.CPULim != "" {
+		lim[v1.ResourceCPU] = resource.MustParse(cr.CPULim)
+	}
+	if cr.MemLim != "" {
+		lim[v1.ResourceMemory] = resource.MustParse(cr.MemLim)
+	}
+	if cr.EphStorLim != "" {
+		lim[v1.ResourceEphemeralStorage] = resource.MustParse(cr.EphStorLim)
+	}
+	if cr.CPUReq != "" {
+		req[v1.ResourceCPU] = resource.MustParse(cr.CPUReq)
+	}
+	if cr.MemReq != "" {
+		req[v1.ResourceMemory] = resource.MustParse(cr.MemReq)
+	}
+	if cr.EphStorReq != "" {
+		req[v1.ResourceEphemeralStorage] = resource.MustParse(cr.EphStorReq)
+	}
+	return &v1.ResourceRequirements{Limits: lim, Requests: req}
+}
+
+func MakeContainerWithResources(name string, r *ContainerResources, command string) v1.Container {
+	var resources v1.ResourceRequirements
+	if r != nil {
+		resources = *r.ResourceRequirements()
+	}
+	return v1.Container{
+		Name:      name,
+		Resources: resources,
+		Image:     imageutils.GetE2EImage(imageutils.BusyBox),
+		Command:   []string{"/bin/sh"},
+		Args:      []string{"-c", command},
+	}
+}
+
+func ConfigureHostPathForPodCgroup(pod *v1.Pod) {
+	if pod.Spec.Volumes == nil {
+		pod.Spec.Volumes = []v1.Volume{}
+	}
+	pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
+		Name: cgroupVolumeName,
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{Path: cgroupFsPath},
+		},
+	})
+	firstContainer := &pod.Spec.Containers[0]
+	if firstContainer.VolumeMounts == nil {
+		firstContainer.VolumeMounts = []v1.VolumeMount{}
+	}
+	firstContainer.VolumeMounts = append(firstContainer.VolumeMounts, v1.VolumeMount{
+		Name:      cgroupVolumeName,
+		MountPath: cgroupMountPath,
+	})
+}
+
+func getPodCgroupPath(f *framework.Framework, pod *v1.Pod, podOnCgroupv2 bool, subsystem string) (string, error) {
+	rootPath := cgroupMountPath
+	if !podOnCgroupv2 {
+		rootPath += "/" + subsystem
+	}
+	// search path for both systemd driver and cgroupfs driver
+	cmd := fmt.Sprintf("find %s -name '*%s*' -o -name '*%s*'", rootPath, strings.ReplaceAll(string(pod.UID), "-", "_"), string(pod.UID))
+	framework.Logf("Namespace %s Pod %s - looking for Pod cgroup directory path: %q", f.Namespace, pod.Name, cmd)
+	podCgPath, stderr, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, []string{"/bin/sh", "-c", cmd}...)
+	if podCgPath == "" {
+		// This command may hit 'No such file or directory' for another cgroup if another test running in parallel has deleted a pod.
+		// We ignore errors if podCgPath is found.
+		if err != nil || len(stderr) > 0 {
+			return "", fmt.Errorf("encountered error while running command: %q, \nerr: %w \nstdErr: %q", cmd, err, stderr)
+		}
+		return "", fmt.Errorf("pod cgroup dirctory not found by command: %q", cmd)
+	}
+	return podCgPath, nil
+}
+
+func getCgroupMemLimitPath(cgPath string, podOnCgroupv2 bool) string {
+	if podOnCgroupv2 {
+		return fmt.Sprintf("%s/%s", cgPath, cgroupv2MemLimitFile)
+	} else {
+		return fmt.Sprintf("%s/memory/%s", cgPath, cgroupMemLimitFile)
+	}
+}
+
+func getCgroupCPULimitPath(cgPath string, podOnCgroupv2 bool) string {
+	if podOnCgroupv2 {
+		return fmt.Sprintf("%s/%s", cgPath, cgroupv2CPULimitFile)
+	} else {
+		return fmt.Sprintf("%s/cpu/%s", cgPath, cgroupCPUQuotaFile)
+	}
+}
+
+func getCgroupCPURequestPath(cgPath string, podOnCgroupv2 bool) string {
+	if podOnCgroupv2 {
+		return fmt.Sprintf("%s/%s", cgPath, cgroupv2CPUWeightFile)
+	} else {
+		return fmt.Sprintf("%s/cpu/%s", cgPath, cgroupCPUSharesFile)
+	}
+}
+
+// TODO: Remove the rounded cpu limit values when https://github.com/opencontainers/runc/issues/4622
+// is fixed.
+func getCPULimitCgroupExpectations(cpuLimit *resource.Quantity, podOnCgroupV2 bool) []string {
+	var expectedCPULimits []string
+	milliCPULimit := cpuLimit.MilliValue()
+
+	cpuQuota := kubecm.MilliCPUToQuota(milliCPULimit, kubecm.QuotaPeriod)
+	if cpuLimit.IsZero() {
+		cpuQuota = -1
+	}
+	expectedCPULimits = append(expectedCPULimits, getExpectedCPULimitFromCPUQuota(cpuQuota, podOnCgroupV2))
+
+	if milliCPULimit%10 != 0 && cpuQuota != -1 {
+		roundedCPULimit := (milliCPULimit/10 + 1) * 10
+		cpuQuotaRounded := kubecm.MilliCPUToQuota(roundedCPULimit, kubecm.QuotaPeriod)
+		expectedCPULimits = append(expectedCPULimits, getExpectedCPULimitFromCPUQuota(cpuQuotaRounded, podOnCgroupV2))
+	}
+
+	return expectedCPULimits
+}
+
+func getExpectedCPULimitFromCPUQuota(cpuQuota int64, podOnCgroupV2 bool) string {
+	expectedCPULimitString := strconv.FormatInt(cpuQuota, 10)
+	if podOnCgroupV2 {
+		if expectedCPULimitString == "-1" {
+			expectedCPULimitString = "max"
+		}
+		expectedCPULimitString = fmt.Sprintf("%s %d", expectedCPULimitString, kubecm.QuotaPeriod)
+	}
+	return expectedCPULimitString
+}
+
+func getExpectedCPUShares(rr *v1.ResourceRequirements, podOnCgroupv2 bool) int64 {
+	cpuRequest := rr.Requests.Cpu()
+	cpuLimit := rr.Limits.Cpu()
+	var shares int64
+	if cpuRequest.IsZero() && !cpuLimit.IsZero() {
+		shares = int64(kubecm.MilliCPUToShares(cpuLimit.MilliValue()))
+	} else {
+		shares = int64(kubecm.MilliCPUToShares(cpuRequest.MilliValue()))
+	}
+	if podOnCgroupv2 {
+		// TODO: This fomula should be a shared function.
+		return 1 + ((shares-2)*9999)/262142
+	} else {
+		return shares
+	}
+}
+
+func getExpectedMemLimitString(rr *v1.ResourceRequirements, podOnCgroupv2 bool) string {
+	expectedMemLimitInBytes := rr.Limits.Memory().Value()
+	expectedMemLimitString := strconv.FormatInt(expectedMemLimitInBytes, 10)
+	if podOnCgroupv2 && expectedMemLimitString == "0" {
+		expectedMemLimitString = "max"
+	}
+	return expectedMemLimitString
+}
+
+func verifyContainerCPUWeight(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
+	cpuWeightCgPath := getCgroupCPURequestPath(cgroupFsPath, podOnCgroupv2)
+	expectedCPUShares := getExpectedCPUShares(expectedResources, podOnCgroupv2)
+	if err := VerifyCgroupValue(f, pod, containerName, cpuWeightCgPath, strconv.FormatInt(expectedCPUShares, 10)); err != nil {
+		return fmt.Errorf("failed to verify cpu request cgroup value: %w", err)
+	}
+	return nil
+}
+
+func VerifyContainerCPULimit(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
+	cpuLimCgPath := getCgroupCPULimitPath(cgroupFsPath, podOnCgroupv2)
+	expectedCPULimits := getCPULimitCgroupExpectations(expectedResources.Limits.Cpu(), podOnCgroupv2)
+	if err := VerifyCgroupValue(f, pod, containerName, cpuLimCgPath, expectedCPULimits...); err != nil {
+		return fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)
+	}
+	return nil
+}
+
+func VerifyContainerMemoryLimit(f *framework.Framework, pod *v1.Pod, containerName string, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
+	memLimCgPath := getCgroupMemLimitPath(cgroupFsPath, podOnCgroupv2)
+	expectedMemLim := getExpectedMemLimitString(expectedResources, podOnCgroupv2)
+	if expectedMemLim == "0" {
+		return nil
+	}
+	if err := VerifyCgroupValue(f, pod, containerName, memLimCgPath, expectedMemLim); err != nil {
+		return fmt.Errorf("failed to verify memory limit cgroup value: %w", err)
+	}
+	return nil
+}
+
+func VerifyContainerCgroupValues(f *framework.Framework, pod *v1.Pod, tc *v1.Container, podOnCgroupv2 bool) error {
+	var errs []error
+	errs = append(errs, VerifyContainerMemoryLimit(f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
+	errs = append(errs, VerifyContainerCPULimit(f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
+	errs = append(errs, verifyContainerCPUWeight(f, pod, tc.Name, &tc.Resources, podOnCgroupv2))
+	return utilerrors.NewAggregate(errs)
+}
+
+func verifyPodCPUWeight(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
+	podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "cpu")
+	if err != nil {
+		if podCgPath, err = getPodCgroupPath(f, pod, podOnCgroupv2, "cpu,cpuacct"); err != nil {
+			return err
+		}
+	}
+
+	var cpuWeightCgPath string
+	if podOnCgroupv2 {
+		cpuWeightCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupv2CPUWeightFile)
+	} else {
+		cpuWeightCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupCPUSharesFile)
+	}
+	expectedCPUShares := getExpectedCPUShares(expectedResources, podOnCgroupv2)
+	if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuWeightCgPath, strconv.FormatInt(expectedCPUShares, 10)); err != nil {
+		return fmt.Errorf("pod cgroup cpu weight verification failed: %w", err)
+	}
+	return nil
+}
+
+func verifyPodCPULimit(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
+	podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "cpu")
+	if err != nil {
+		if podCgPath, err = getPodCgroupPath(f, pod, podOnCgroupv2, "cpu,cpuacct"); err != nil {
+			return err
+		}
+	}
+
+	var cpuLimCgPath string
+	if podOnCgroupv2 {
+		cpuLimCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupv2CPULimitFile)
+	} else {
+		cpuLimCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupCPUQuotaFile)
+	}
+	expectedCPULimits := getCPULimitCgroupExpectations(expectedResources.Limits.Cpu(), podOnCgroupv2)
+	if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuLimCgPath, expectedCPULimits...); err != nil {
+		return fmt.Errorf("pod cgroup cpu limit verification failed: %w", err)
+	}
+	return nil
+}
+
+func verifyPodMemoryLimit(f *framework.Framework, pod *v1.Pod, expectedResources *v1.ResourceRequirements, podOnCgroupv2 bool) error {
+	podCgPath, err := getPodCgroupPath(f, pod, podOnCgroupv2, "memory")
+	if err != nil {
+		return err
+	}
+
+	var memLimCgPath string
+	if podOnCgroupv2 {
+		memLimCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupv2MemLimitFile)
+	} else {
+		memLimCgPath = fmt.Sprintf("%s/%s", podCgPath, cgroupMemLimitFile)
+	}
+	expectedMemLim := getExpectedMemLimitString(expectedResources, podOnCgroupv2)
+	if expectedMemLim == "0" {
+		return nil
+	}
+
+	if err := VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, memLimCgPath, expectedMemLim); err != nil {
+		return fmt.Errorf("pod cgroup memory limit verification failed: %w", err)
+	}
+	return nil
+}
+
+// VerifyPodCgroups verifies pod cgroup is configured on a node as expected.
+func VerifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, info *ContainerResources) error {
+	ginkgo.GinkgoHelper()
+
+	onCgroupV2 := IsPodOnCgroupv2Node(f, pod)
+
+	// Verify cgroup values
+	expectedResources := info.ResourceRequirements()
+	var errs []error
+	errs = append(errs, verifyPodCPUWeight(f, pod, expectedResources, onCgroupV2))
+	errs = append(errs, verifyPodCPULimit(f, pod, expectedResources, onCgroupV2))
+	errs = append(errs, verifyPodMemoryLimit(f, pod, expectedResources, onCgroupV2))
+
+	return utilerrors.NewAggregate(errs)
+}
+
+func BuildPodResourceInfo(podCPURequestMilliValue, podCPULimitMilliValue, podMemoryLimitInBytes int64) ContainerResources {
+	podResourceInfo := ContainerResources{}
+	if podCPURequestMilliValue > 0 {
+		podResourceInfo.CPUReq = fmt.Sprintf("%dm", podCPURequestMilliValue)
+	}
+	if podCPULimitMilliValue > 0 {
+		podResourceInfo.CPULim = fmt.Sprintf("%dm", podCPULimitMilliValue)
+	}
+	if podMemoryLimitInBytes > 0 {
+		podResourceInfo.MemLim = fmt.Sprintf("%d", podMemoryLimitInBytes)
+	}
+	return podResourceInfo
+}
+
+// VerifyCgroupValue verifies that the given cgroup path has the expected value in
+// the specified container of the pod. It execs into the container to retrieve the
+// cgroup value, and ensures that the retrieved cgroup value is equivalent to at
+// least one of the values in expectedCgValues.
+func VerifyCgroupValue(f *framework.Framework, pod *v1.Pod, cName, cgPath string, expectedCgValues ...string) error {
+	cmd := fmt.Sprintf("head -n 1 %s", cgPath)
+	framework.Logf("Namespace %s Pod %s Container %s - looking for one of the expected cgroup values %s in path %s",
+		pod.Namespace, pod.Name, cName, expectedCgValues, cgPath)
+	cgValue, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to read cgroup value %q for container %q: %w", cgPath, cName, err)
+	}
+	cgValue = strings.Trim(cgValue, "\n")
+
+	if err := framework.Gomega().Expect(cgValue).To(gomega.BeElementOf(expectedCgValues)); err != nil {
+		return fmt.Errorf("value of cgroup %q for container %q was %q; expected one of %q", cgPath, cName, cgValue, expectedCgValues)
+	}
+
+	return nil
+}
+
+// VerifyOomScoreAdjValue verifies that oom_score_adj for pid 1 (pidof init/systemd -> app)
+// has the expected value in specified container of the pod. It execs into the container,
+// reads the oom_score_adj value from procfs, and compares it against the expected value.
+func VerifyOomScoreAdjValue(f *framework.Framework, pod *v1.Pod, cName, expectedOomScoreAdj string) error {
+	cmd := "cat /proc/1/oom_score_adj"
+	framework.Logf("Namespace %s Pod %s Container %s - looking for oom_score_adj value %s",
+		pod.Namespace, pod.Name, cName, expectedOomScoreAdj)
+	oomScoreAdj, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to find expected value %s for container app process", expectedOomScoreAdj)
+	}
+	oomScoreAdj = strings.Trim(oomScoreAdj, "\n")
+	if oomScoreAdj != expectedOomScoreAdj {
+		return fmt.Errorf("oom_score_adj value %s not equal to expected %s", oomScoreAdj, expectedOomScoreAdj)
+	}
+	return nil
+}
+
+// IsPodOnCgroupv2Node checks whether the pod is running on cgroupv2 node.
+// TODO: Deduplicate this function with NPD cluster e2e test:
+// https://github.com/kubernetes/kubernetes/blob/2049360379bcc5d6467769cef112e6e492d3d2f0/test/e2e/node/node_problem_detector.go#L369
+func IsPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) (result bool) {
+	podOnCgroupv2NodeMutex.Lock()
+	defer podOnCgroupv2NodeMutex.Unlock()
+	if podOnCgroupv2Node != nil {
+		return *podOnCgroupv2Node
+	}
+	defer func() {
+		podOnCgroupv2Node = &result
+	}()
+
+	cmd := "mount -t cgroup2"
+	out, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd)
+	if err != nil {
+		return false
+	}
+	// Some tests mount host cgroup using HostPath for verifying pod cgroup values.
+	// In this case, "<mount path>/unified" is detected by "mount -t cgroup2" if cgroup hybrid mode is configured on the host.
+	// So, we need to see if "/sys/fs/cgroup" is contained in the output.
+	return strings.Contains(out, "/sys/fs/cgroup")
+}

test/e2e/common/node/framework/cgroups/cgroups_test.go

@@ -17,7 +17,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package podresize
+package cgroups
 
 import (
 	"testing"
@@ -71,13 +71,9 @@ func TestGetCPULimitCgroupExpectations(t *testing.T) {
 		},
 	}
 
-	originalPodOnCgroupv2Node := podOnCgroupv2Node
-	t.Cleanup(func() { podOnCgroupv2Node = originalPodOnCgroupv2Node })
-
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			podOnCgroupv2Node = &tc.podOnCgroupv2Node
-			actual := GetCPULimitCgroupExpectations(tc.cpuLimit)
+			actual := getCPULimitCgroupExpectations(tc.cpuLimit, tc.podOnCgroupv2Node)
 			assert.Equal(t, tc.expected, actual)
 		})
 	}

test/e2e/common/node/framework/podresize/resize.go

@@ -25,85 +25,26 @@ import (
 	"strings"
 
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	helpers "k8s.io/component-helpers/resource"
 	"k8s.io/kubectl/pkg/util/podutils"
-	kubecm "k8s.io/kubernetes/pkg/kubelet/cm"
 	kubeqos "k8s.io/kubernetes/pkg/kubelet/qos"
+	"k8s.io/kubernetes/test/e2e/common/node/framework/cgroups"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-	imageutils "k8s.io/kubernetes/test/utils/image"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 )
 
 const (
-	CgroupCPUPeriod            string = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
-	CgroupCPUShares            string = "/sys/fs/cgroup/cpu/cpu.shares"
-	CgroupCPUQuota             string = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
-	CgroupMemLimit             string = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
-	Cgroupv2MemLimit           string = "/sys/fs/cgroup/memory.max"
-	Cgroupv2MemRequest         string = "/sys/fs/cgroup/memory.min"
-	Cgroupv2CPULimit           string = "/sys/fs/cgroup/cpu.max"
-	Cgroupv2CPURequest         string = "/sys/fs/cgroup/cpu.weight"
-	CPUPeriod                  string = "100000"
 	MinContainerRuntimeVersion string = "1.6.9"
 )
 
-var (
-	podOnCgroupv2Node *bool
-)
-
-type ContainerResources struct {
-	CPUReq              string
-	CPULim              string
-	MemReq              string
-	MemLim              string
-	EphStorReq          string
-	EphStorLim          string
-	ExtendedResourceReq string
-	ExtendedResourceLim string
-}
-
-func (cr *ContainerResources) ResourceRequirements() *v1.ResourceRequirements {
-	if cr == nil {
-		return nil
-	}
-
-	var lim, req v1.ResourceList
-	if cr.CPULim != "" || cr.MemLim != "" || cr.EphStorLim != "" {
-		lim = make(v1.ResourceList)
-	}
-	if cr.CPUReq != "" || cr.MemReq != "" || cr.EphStorReq != "" {
-		req = make(v1.ResourceList)
-	}
-	if cr.CPULim != "" {
-		lim[v1.ResourceCPU] = resource.MustParse(cr.CPULim)
-	}
-	if cr.MemLim != "" {
-		lim[v1.ResourceMemory] = resource.MustParse(cr.MemLim)
-	}
-	if cr.EphStorLim != "" {
-		lim[v1.ResourceEphemeralStorage] = resource.MustParse(cr.EphStorLim)
-	}
-	if cr.CPUReq != "" {
-		req[v1.ResourceCPU] = resource.MustParse(cr.CPUReq)
-	}
-	if cr.MemReq != "" {
-		req[v1.ResourceMemory] = resource.MustParse(cr.MemReq)
-	}
-	if cr.EphStorReq != "" {
-		req[v1.ResourceEphemeralStorage] = resource.MustParse(cr.EphStorReq)
-	}
-	return &v1.ResourceRequirements{Limits: lim, Requests: req}
-}
-
 type ResizableContainerInfo struct {
 	Name          string
-	Resources     *ContainerResources
+	Resources     *cgroups.ContainerResources
 	CPUPolicy     *v1.ResourceResizeRestartPolicy
 	MemPolicy     *v1.ResourceResizeRestartPolicy
 	RestartCount  int32
@@ -133,10 +74,7 @@ type patchSpec struct {
 	} `json:"spec"`
 }
 
-func getTestResourceInfo(tcInfo ResizableContainerInfo) (res v1.ResourceRequirements, resizePol []v1.ContainerResizePolicy) {
-	if tcInfo.Resources != nil {
-		res = *tcInfo.Resources.ResourceRequirements()
-	}
+func getTestResizePolicy(tcInfo ResizableContainerInfo) (resizePol []v1.ContainerResizePolicy) {
 	if tcInfo.CPUPolicy != nil {
 		cpuPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceCPU, RestartPolicy: *tcInfo.CPUPolicy}
 		resizePol = append(resizePol, cpuPol)
@@ -145,21 +83,14 @@ func getTestResourceInfo(tcInfo ResizableContainerInfo) (res v1.ResourceRequirem
 		memPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceMemory, RestartPolicy: *tcInfo.MemPolicy}
 		resizePol = append(resizePol, memPol)
 	}
-	return res, resizePol
+	return resizePol
 }
 
 func makeResizableContainer(tcInfo ResizableContainerInfo) v1.Container {
 	cmd := "grep Cpus_allowed_list /proc/self/status | cut -f2 && sleep 1d"
-	res, resizePol := getTestResourceInfo(tcInfo)
-
-	tc := v1.Container{
-		Name:         tcInfo.Name,
-		Image:        imageutils.GetE2EImage(imageutils.BusyBox),
-		Command:      []string{"/bin/sh"},
-		Args:         []string{"-c", cmd},
-		Resources:    res,
-		ResizePolicy: resizePol,
-	}
+	resizePol := getTestResizePolicy(tcInfo)
+	tc := cgroups.MakeContainerWithResources(tcInfo.Name, tcInfo.Resources, cmd)
+	tc.ResizePolicy = resizePol
 	if tcInfo.RestartPolicy != "" {
 		tc.RestartPolicy = &tcInfo.RestartPolicy
 	}
@@ -302,65 +233,13 @@ func verifyPodContainersStatusResources(gotCtrStatuses []v1.ContainerStatus, wan
 
 func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework, pod *v1.Pod, tcInfo []ResizableContainerInfo) error {
 	ginkgo.GinkgoHelper()
-	if podOnCgroupv2Node == nil {
-		value := e2epod.IsPodOnCgroupv2Node(f, pod)
-		podOnCgroupv2Node = &value
-	}
-	cgroupMemLimit := Cgroupv2MemLimit
-	cgroupCPULimit := Cgroupv2CPULimit
-	cgroupCPURequest := Cgroupv2CPURequest
-	if !*podOnCgroupv2Node {
-		cgroupMemLimit = CgroupMemLimit
-		cgroupCPULimit = CgroupCPUQuota
-		cgroupCPURequest = CgroupCPUShares
-	}
+
+	onCgroupv2 := cgroups.IsPodOnCgroupv2Node(f, pod)
 
 	var errs []error
 	for _, ci := range tcInfo {
-		if ci.Resources == nil {
-			continue
-		}
 		tc := makeResizableContainer(ci)
-		if tc.Resources.Limits != nil || tc.Resources.Requests != nil {
-			var expectedCPUShares int64
-			var expectedMemLimitString string
-			expectedMemLimitInBytes := tc.Resources.Limits.Memory().Value()
-			cpuRequest := tc.Resources.Requests.Cpu()
-			cpuLimit := tc.Resources.Limits.Cpu()
-			if cpuRequest.IsZero() && !cpuLimit.IsZero() {
-				expectedCPUShares = int64(kubecm.MilliCPUToShares(cpuLimit.MilliValue()))
-			} else {
-				expectedCPUShares = int64(kubecm.MilliCPUToShares(cpuRequest.MilliValue()))
-			}
-
-			expectedCPULimits := GetCPULimitCgroupExpectations(cpuLimit)
-			expectedMemLimitString = strconv.FormatInt(expectedMemLimitInBytes, 10)
-			if *podOnCgroupv2Node {
-				if expectedMemLimitString == "0" {
-					expectedMemLimitString = "max"
-				}
-				// convert cgroup v1 cpu.shares value to cgroup v2 cpu.weight value
-				// https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2
-				expectedCPUShares = int64(1 + ((expectedCPUShares-2)*9999)/262142)
-			}
-
-			if expectedMemLimitString != "0" {
-				if err := e2epod.VerifyCgroupValue(f, pod, ci.Name, cgroupMemLimit, expectedMemLimitString); err != nil {
-					errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
-				}
-			}
-
-			if err := e2epod.VerifyCgroupValue(f, pod, ci.Name, cgroupCPULimit, expectedCPULimits...); err != nil {
-				errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err))
-			}
-
-			if err := e2epod.VerifyCgroupValue(f, pod, ci.Name, cgroupCPURequest, strconv.FormatInt(expectedCPUShares, 10)); err != nil {
-				errs = append(errs, fmt.Errorf("failed to verify cpu request cgroup value: %w", err))
-			}
-
-			// TODO(vinaykul,InPlacePodVerticalScaling): Verify oom_score_adj when runc adds support for updating it
-			// See https://github.com/opencontainers/runc/pull/4669
-		}
+		errs = append(errs, cgroups.VerifyContainerCgroupValues(f, pod, &tc, onCgroupv2))
 	}
 	return utilerrors.NewAggregate(errs)
 }
@@ -416,7 +295,7 @@ func verifyOomScoreAdj(f *framework.Framework, pod *v1.Pod, containerName string
 	oomScoreAdj := kubeqos.GetContainerOOMScoreAdjust(pod, container, int64(nodeMemoryCapacity.Value()))
 	expectedOomScoreAdj := strconv.FormatInt(int64(oomScoreAdj), 10)
 
-	return e2epod.VerifyOomScoreAdjValue(f, pod, container.Name, expectedOomScoreAdj)
+	return cgroups.VerifyOomScoreAdjValue(f, pod, container.Name, expectedOomScoreAdj)
 }
 
 func WaitForPodResizeActuation(ctx context.Context, f *framework.Framework, podClient *e2epod.PodClient, pod *v1.Pod, expectedContainers []ResizableContainerInfo) *v1.Pod {
@@ -544,35 +423,3 @@ func formatErrors(err error) error {
 	}
 	return fmt.Errorf("[\n%s\n]", strings.Join(errStrings, ",\n"))
 }
-
-// TODO: Remove the rounded cpu limit values when https://github.com/opencontainers/runc/issues/4622
-// is fixed.
-func GetCPULimitCgroupExpectations(cpuLimit *resource.Quantity) []string {
-	var expectedCPULimits []string
-	milliCPULimit := cpuLimit.MilliValue()
-
-	cpuQuota := kubecm.MilliCPUToQuota(milliCPULimit, kubecm.QuotaPeriod)
-	if cpuLimit.IsZero() {
-		cpuQuota = -1
-	}
-	expectedCPULimits = append(expectedCPULimits, getExpectedCPULimitFromCPUQuota(cpuQuota))
-
-	if milliCPULimit%10 != 0 && cpuQuota != -1 {
-		roundedCPULimit := (milliCPULimit/10 + 1) * 10
-		cpuQuotaRounded := kubecm.MilliCPUToQuota(roundedCPULimit, kubecm.QuotaPeriod)
-		expectedCPULimits = append(expectedCPULimits, getExpectedCPULimitFromCPUQuota(cpuQuotaRounded))
-	}
-
-	return expectedCPULimits
-}
-
-func getExpectedCPULimitFromCPUQuota(cpuQuota int64) string {
-	expectedCPULimitString := strconv.FormatInt(cpuQuota, 10)
-	if *podOnCgroupv2Node {
-		if expectedCPULimitString == "-1" {
-			expectedCPULimitString = "max"
-		}
-		expectedCPULimitString = fmt.Sprintf("%s %s", expectedCPULimitString, CPUPeriod)
-	}
-	return expectedCPULimitString
-}

test/e2e/common/node/pod_level_resources.go

@@ -20,17 +20,14 @@ import (
 	"context"
 	"fmt"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
-	kubecm "k8s.io/kubernetes/pkg/kubelet/cm"
-	"k8s.io/kubernetes/test/e2e/common/node/framework/podresize"
+	"k8s.io/kubernetes/test/e2e/common/node/framework/cgroups"
 	"k8s.io/kubernetes/test/e2e/feature"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
@@ -40,16 +37,8 @@ import (
 	admissionapi "k8s.io/pod-security-admission/api"
 )
 
-const (
-	cgroupv2CPUWeight string = "cpu.weight"
-	cgroupv2CPULimit  string = "cpu.max"
-	cgroupv2MemLimit  string = "memory.max"
-	cgroupFsPath      string = "/sys/fs/cgroup"
-	mountPath         string = "/sysfscgroup"
-)
-
 var (
-	cmd = []string{"/bin/sh", "-c", "sleep 1d"}
+	cmd = e2epod.InfiniteSleepCommand
 )
 
 var _ = SIGDescribe("Pod Level Resources", framework.WithSerial(), feature.PodLevelResources, func() {
@@ -78,6 +67,7 @@ var _ = SIGDescribe("Pod Level Resources", framework.WithSerial(), feature.PodLe
 // the label in the test job, to tun this test on a node with cgroupv2.
 func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool {
 	podClient := e2epod.NewPodClient(f)
+	containerResources := cgroups.ContainerResources{CPULim: "1m", MemReq: "1Mi"}
 	cgroupv2Testpod := &v1.Pod{
 		ObjectMeta: makeObjectMetadata("cgroupv2-check", f.Namespace.Name),
 		Spec: v1.PodSpec{
@@ -85,8 +75,8 @@ func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool {
 				{
 					Name:      "cgroupv2-check",
 					Image:     imageutils.GetE2EImage(imageutils.BusyBox),
-					Command:   cmd,
-					Resources: getResourceRequirements(&resourceInfo{CPULim: "1m", MemReq: "1Mi"}),
+					Command:   []string{"/bin/sh", "-c", cmd},
+					Resources: *containerResources.ResourceRequirements(),
 				},
 			},
 		},
@@ -99,7 +89,7 @@ func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool {
 		framework.ExpectNoError(delErr, "failed to delete pod %s", delErr)
 	}()
 
-	return e2epod.IsPodOnCgroupv2Node(f, pod)
+	return cgroups.IsPodOnCgroupv2Node(f, pod)
 }
 
 func makeObjectMetadata(name, namespace string) metav1.ObjectMeta {
@@ -111,62 +101,14 @@ func makeObjectMetadata(name, namespace string) metav1.ObjectMeta {
 
 type containerInfo struct {
 	Name      string
-	Resources *resourceInfo
-}
-type resourceInfo struct {
-	CPUReq string
-	CPULim string
-	MemReq string
-	MemLim string
+	Resources *cgroups.ContainerResources
 }
 
-func makeContainer(info containerInfo) v1.Container {
-	cmd := []string{"/bin/sh", "-c", "sleep 1d"}
-	res := getResourceRequirements(info.Resources)
-	return v1.Container{
-		Name:      info.Name,
-		Command:   cmd,
-		Resources: res,
-		Image:     imageutils.GetE2EImage(imageutils.BusyBox),
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "sysfscgroup",
-				MountPath: mountPath,
-			},
-		},
-	}
-}
-
-func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements {
-	var res v1.ResourceRequirements
-	if info != nil {
-		if info.CPUReq != "" || info.MemReq != "" {
-			res.Requests = make(v1.ResourceList)
-		}
-		if info.CPUReq != "" {
-			res.Requests[v1.ResourceCPU] = resource.MustParse(info.CPUReq)
-		}
-		if info.MemReq != "" {
-			res.Requests[v1.ResourceMemory] = resource.MustParse(info.MemReq)
-		}
-
-		if info.CPULim != "" || info.MemLim != "" {
-			res.Limits = make(v1.ResourceList)
-		}
-		if info.CPULim != "" {
-			res.Limits[v1.ResourceCPU] = resource.MustParse(info.CPULim)
-		}
-		if info.MemLim != "" {
-			res.Limits[v1.ResourceMemory] = resource.MustParse(info.MemLim)
-		}
-	}
-	return res
-}
-
-func makePod(metadata *metav1.ObjectMeta, podResources *resourceInfo, containers []containerInfo) *v1.Pod {
+func makePod(metadata *metav1.ObjectMeta, podResources *cgroups.ContainerResources, containers []containerInfo) *v1.Pod {
 	var testContainers []v1.Container
 	for _, container := range containers {
-		testContainers = append(testContainers, makeContainer(container))
+		c := cgroups.MakeContainerWithResources(container.Name, container.Resources, cmd)
+		testContainers = append(testContainers, c)
 	}
 
 	pod := &v1.Pod{
@@ -174,33 +116,25 @@ func makePod(metadata *metav1.ObjectMeta, podResources *resourceInfo, containers
 
 		Spec: v1.PodSpec{
 			Containers: testContainers,
-			Volumes: []v1.Volume{
-				{
-					Name: "sysfscgroup",
-					VolumeSource: v1.VolumeSource{
-						HostPath: &v1.HostPathVolumeSource{Path: cgroupFsPath},
-					},
-				},
-			},
 		},
 	}
+	cgroups.ConfigureHostPathForPodCgroup(pod)
 
 	if podResources != nil {
-		res := getResourceRequirements(podResources)
-		pod.Spec.Resources = &res
+		res := podResources.ResourceRequirements()
+		pod.Spec.Resources = res
 	}
 
 	return pod
 }
 
-func verifyPodResources(gotPod v1.Pod, inputInfo, expectedInfo *resourceInfo) {
+func verifyPodResources(gotPod v1.Pod, inputInfo, expectedInfo *cgroups.ContainerResources) {
 	ginkgo.GinkgoHelper()
 	var expectedResources *v1.ResourceRequirements
 	// expectedResources will be nil if pod-level resources are not set in the test
 	// case input.
 	if inputInfo != nil {
-		resourceInfo := getResourceRequirements(expectedInfo)
-		expectedResources = &resourceInfo
+		expectedResources = expectedInfo.ResourceRequirements()
 	}
 	gomega.Expect(expectedResources).To(gomega.Equal(gotPod.Spec.Resources))
 }
@@ -210,45 +144,6 @@ func verifyQoS(gotPod v1.Pod, expectedQoS v1.PodQOSClass) {
 	gomega.Expect(expectedQoS).To(gomega.Equal(gotPod.Status.QOSClass))
 }
 
-// TODO(ndixita): dedup the conversion logic in pod resize test and move to helpers/utils.
-func verifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, info *resourceInfo) error {
-	ginkgo.GinkgoHelper()
-	cmd := fmt.Sprintf("find %s -name '*%s*'", mountPath, strings.ReplaceAll(string(pod.UID), "-", "_"))
-	framework.Logf("Namespace %s Pod %s - looking for Pod cgroup directory path: %q", f.Namespace, pod.Name, cmd)
-	podCgPath, stderr, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, []string{"/bin/sh", "-c", cmd}...)
-	if err != nil || len(stderr) > 0 {
-		return fmt.Errorf("encountered error while running command: %q, \nerr: %w \nstdErr: %q", cmd, err, stderr)
-	}
-
-	expectedResources := getResourceRequirements(info)
-	cpuWeightCgPath := fmt.Sprintf("%s/%s", podCgPath, cgroupv2CPUWeight)
-	expectedCPUShares := int64(kubecm.MilliCPUToShares(expectedResources.Requests.Cpu().MilliValue()))
-	expectedCPUShares = int64(1 + ((expectedCPUShares-2)*9999)/262142)
-	// convert cgroup v1 cpu.shares value to cgroup v2 cpu.weight value
-	// https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2
-	var errs []error
-	err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuWeightCgPath, strconv.FormatInt(expectedCPUShares, 10))
-	if err != nil {
-		errs = append(errs, fmt.Errorf("failed to verify cpu request cgroup value: %w", err))
-	}
-
-	cpuLimCgPath := fmt.Sprintf("%s/%s", podCgPath, cgroupv2CPULimit)
-	expectedCPULimits := podresize.GetCPULimitCgroupExpectations(expectedResources.Limits.Cpu())
-
-	err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuLimCgPath, expectedCPULimits...)
-	if err != nil {
-		errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err))
-	}
-
-	memLimCgPath := fmt.Sprintf("%s/%s", podCgPath, cgroupv2MemLimit)
-	expectedMemLim := strconv.FormatInt(expectedResources.Limits.Memory().Value(), 10)
-	err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, memLimCgPath, expectedMemLim)
-	if err != nil {
-		errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
-	}
-	return utilerrors.NewAggregate(errs)
-}
-
 func podLevelResourcesTests(f *framework.Framework) {
 	type expectedPodConfig struct {
 		qos v1.PodQOSClass
@@ -257,12 +152,12 @@ func podLevelResourcesTests(f *framework.Framework) {
 		// are specified, totalPodResources is equal to pod-level resources.
 		// Otherwise, it is calculated by aggregating resource requests and
 		// limits from all containers within the pod..
-		totalPodResources *resourceInfo
+		totalPodResources *cgroups.ContainerResources
 	}
 
 	type testCase struct {
 		name         string
-		podResources *resourceInfo
+		podResources *cgroups.ContainerResources
 		containers   []containerInfo
 		expected     expectedPodConfig
 	}
@@ -271,81 +166,81 @@ func podLevelResourcesTests(f *framework.Framework) {
 		{
 			name: "Guaranteed QoS pod with container resources",
 			containers: []containerInfo{
-				{Name: "c1", Resources: &resourceInfo{CPUReq: "50m", CPULim: "50m", MemReq: "70Mi", MemLim: "70Mi"}},
-				{Name: "c2", Resources: &resourceInfo{CPUReq: "70m", CPULim: "70m", MemReq: "50Mi", MemLim: "50Mi"}},
+				{Name: "c1", Resources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "50m", MemReq: "70Mi", MemLim: "70Mi"}},
+				{Name: "c2", Resources: &cgroups.ContainerResources{CPUReq: "70m", CPULim: "70m", MemReq: "50Mi", MemLim: "50Mi"}},
 			},
 			expected: expectedPodConfig{
 				qos:               v1.PodQOSGuaranteed,
-				totalPodResources: &resourceInfo{CPUReq: "120m", CPULim: "120m", MemReq: "120Mi", MemLim: "120Mi"},
+				totalPodResources: &cgroups.ContainerResources{CPUReq: "120m", CPULim: "120m", MemReq: "120Mi", MemLim: "120Mi"},
 			},
 		},
 		{
 			name:         "Guaranteed QoS pod, no container resources",
-			podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
+			podResources: &cgroups.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
 			containers:   []containerInfo{{Name: "c1"}, {Name: "c2"}},
 			expected: expectedPodConfig{
 				qos:               v1.PodQOSGuaranteed,
-				totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
+				totalPodResources: &cgroups.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
 			},
 		},
 		{
 			name:         "Guaranteed QoS pod with container resources",
-			podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
+			podResources: &cgroups.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
 			containers: []containerInfo{
-				{Name: "c1", Resources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}},
-				{Name: "c2", Resources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}},
+				{Name: "c1", Resources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}},
+				{Name: "c2", Resources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}},
 			},
 			expected: expectedPodConfig{
 				qos:               v1.PodQOSGuaranteed,
-				totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
+				totalPodResources: &cgroups.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
 			},
 		},
 		{
 			name:         "Guaranteed QoS pod, 1 container with resources",
-			podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
+			podResources: &cgroups.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
 			containers: []containerInfo{
-				{Name: "c1", Resources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}},
+				{Name: "c1", Resources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}},
 				{Name: "c2"},
 			},
 			expected: expectedPodConfig{
 				qos:               v1.PodQOSGuaranteed,
-				totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
+				totalPodResources: &cgroups.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"},
 			},
 		},
 		{
 			name:         "Burstable QoS pod, no container resources",
-			podResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
+			podResources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
 			containers: []containerInfo{
 				{Name: "c1"},
 				{Name: "c2"},
 			},
 			expected: expectedPodConfig{
 				qos:               v1.PodQOSBurstable,
-				totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
+				totalPodResources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
 			},
 		},
 		{
 			name:         "Burstable QoS pod with container resources",
-			podResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
+			podResources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
 			containers: []containerInfo{
-				{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "100m", MemReq: "20Mi", MemLim: "100Mi"}},
-				{Name: "c2", Resources: &resourceInfo{CPUReq: "30m", CPULim: "100m", MemReq: "30Mi", MemLim: "100Mi"}},
+				{Name: "c1", Resources: &cgroups.ContainerResources{CPUReq: "20m", CPULim: "100m", MemReq: "20Mi", MemLim: "100Mi"}},
+				{Name: "c2", Resources: &cgroups.ContainerResources{CPUReq: "30m", CPULim: "100m", MemReq: "30Mi", MemLim: "100Mi"}},
 			},
 			expected: expectedPodConfig{
 				qos:               v1.PodQOSBurstable,
-				totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
+				totalPodResources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
 			},
 		},
 		{
 			name:         "Burstable QoS pod, 1 container with resources",
-			podResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
+			podResources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
 			containers: []containerInfo{
-				{Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}},
+				{Name: "c1", Resources: &cgroups.ContainerResources{CPUReq: "20m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}},
 				{Name: "c2"},
 			},
 			expected: expectedPodConfig{
 				qos:               v1.PodQOSBurstable,
-				totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
+				totalPodResources: &cgroups.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"},
 			},
 		},
 	}
@@ -366,7 +261,7 @@ func podLevelResourcesTests(f *framework.Framework) {
 			verifyQoS(*pod, tc.expected.qos)
 
 			ginkgo.By("verifying pod cgroup values")
-			err := verifyPodCgroups(ctx, f, pod, tc.expected.totalPodResources)
+			err := cgroups.VerifyPodCgroups(ctx, f, pod, tc.expected.totalPodResources)
 			framework.ExpectNoError(err, "failed to verify pod's cgroup values: %v", err)
 
 			ginkgo.By("verifying containers cgroup limits are same as pod container's cgroup limits")
@@ -385,8 +280,7 @@ func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error {
 	for _, container := range pod.Spec.Containers {
 		if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Memory() != nil &&
 			container.Resources.Limits.Memory() == nil {
-			expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10)
-			err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit)
+			err := cgroups.VerifyContainerMemoryLimit(f, pod, container.Name, &container.Resources, true)
 			if err != nil {
 				errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err))
 			}
@@ -394,8 +288,7 @@ func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error {
 
 		if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Cpu() != nil &&
 			container.Resources.Limits.Cpu() == nil {
-			expectedCPULimits := podresize.GetCPULimitCgroupExpectations(pod.Spec.Resources.Limits.Cpu())
-			err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimits...)
+			err := cgroups.VerifyContainerCPULimit(f, pod, container.Name, &container.Resources, true)
 			if err != nil {
 				errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err))
 			}

test/e2e/framework/pod/utils.go

@@ -18,7 +18,6 @@ package pod
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
@@ -290,54 +289,3 @@ func FindContainerStatusInPod(pod *v1.Pod, containerName string) *v1.ContainerSt
 	}
 	return nil
 }
-
-// VerifyCgroupValue verifies that the given cgroup path has the expected value in
-// the specified container of the pod. It execs into the container to retrieve the
-// cgroup value, and ensures that the retrieved cgroup value is equivalent to at
-// least one of the values in expectedCgValues.
-func VerifyCgroupValue(f *framework.Framework, pod *v1.Pod, cName, cgPath string, expectedCgValues ...string) error {
-	cmd := fmt.Sprintf("head -n 1 %s", cgPath)
-	framework.Logf("Namespace %s Pod %s Container %s - looking for one of the expected cgroup values %s in path %s",
-		pod.Namespace, pod.Name, cName, expectedCgValues, cgPath)
-	cgValue, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd)
-	if err != nil {
-		return fmt.Errorf("failed to read cgroup value %q for container %q: %w", cgPath, cName, err)
-	}
-	cgValue = strings.Trim(cgValue, "\n")
-
-	if err := framework.Gomega().Expect(cgValue).To(gomega.BeElementOf(expectedCgValues)); err != nil {
-		return fmt.Errorf("value of cgroup %q for container %q was %q; expected one of %q", cgPath, cName, cgValue, expectedCgValues)
-	}
-
-	return nil
-}
-
-// VerifyOomScoreAdjValue verifies that oom_score_adj for pid 1 (pidof init/systemd -> app)
-// has the expected value in specified container of the pod. It execs into the container,
-// reads the oom_score_adj value from procfs, and compares it against the expected value.
-func VerifyOomScoreAdjValue(f *framework.Framework, pod *v1.Pod, cName, expectedOomScoreAdj string) error {
-	cmd := "cat /proc/1/oom_score_adj"
-	framework.Logf("Namespace %s Pod %s Container %s - looking for oom_score_adj value %s",
-		pod.Namespace, pod.Name, cName, expectedOomScoreAdj)
-	oomScoreAdj, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd)
-	if err != nil {
-		return fmt.Errorf("failed to find expected value %s for container app process", expectedOomScoreAdj)
-	}
-	oomScoreAdj = strings.Trim(oomScoreAdj, "\n")
-	if oomScoreAdj != expectedOomScoreAdj {
-		return fmt.Errorf("oom_score_adj value %s not equal to expected %s", oomScoreAdj, expectedOomScoreAdj)
-	}
-	return nil
-}
-
-// IsPodOnCgroupv2Node checks whether the pod is running on cgroupv2 node.
-// TODO: Deduplicate this function with NPD cluster e2e test:
-// https://github.com/kubernetes/kubernetes/blob/2049360379bcc5d6467769cef112e6e492d3d2f0/test/e2e/node/node_problem_detector.go#L369
-func IsPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) bool {
-	cmd := "mount -t cgroup2"
-	out, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd)
-	if err != nil {
-		return false
-	}
-	return len(out) != 0
-}

test/e2e/node/pod_resize.go

@@ -30,6 +30,7 @@ import (
 	helpers "k8s.io/component-helpers/resource"
 	resourceapi "k8s.io/kubernetes/pkg/api/v1/resource"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/test/e2e/common/node/framework/cgroups"
 	"k8s.io/kubernetes/test/e2e/common/node/framework/podresize"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
@@ -123,7 +124,7 @@ func doPodResizeAdmissionPluginsTests(f *framework.Framework) {
 			containers := []podresize.ResizableContainerInfo{
 				{
 					Name:      "c1",
-					Resources: &podresize.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"},
+					Resources: &cgroups.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"},
 				},
 			}
 			patchString := `{"spec":{"containers":[
@@ -132,7 +133,7 @@ func doPodResizeAdmissionPluginsTests(f *framework.Framework) {
 			expected := []podresize.ResizableContainerInfo{
 				{
 					Name:      "c1",
-					Resources: &podresize.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "400Mi", MemLim: "400Mi"},
+					Resources: &cgroups.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "400Mi", MemLim: "400Mi"},
 				},
 			}
 			patchStringExceedCPU := `{"spec":{"containers":[
@@ -264,13 +265,13 @@ func doPodResizeSchedulerTests(f *framework.Framework) {
 		c1 := []podresize.ResizableContainerInfo{
 			{
 				Name:      "c1",
-				Resources: &podresize.ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()},
+				Resources: &cgroups.ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()},
 			},
 		}
 		c2 := []podresize.ResizableContainerInfo{
 			{
 				Name:      "c2",
-				Resources: &podresize.ContainerResources{CPUReq: testPod2CPUQuantity.String(), CPULim: testPod2CPUQuantity.String()},
+				Resources: &cgroups.ContainerResources{CPUReq: testPod2CPUQuantity.String(), CPULim: testPod2CPUQuantity.String()},
 			},
 		}
 		patchTestpod2ToFitNode := fmt.Sprintf(`{
@@ -328,7 +329,7 @@ func doPodResizeSchedulerTests(f *framework.Framework) {
 		c3 := []podresize.ResizableContainerInfo{
 			{
 				Name:      "c3",
-				Resources: &podresize.ContainerResources{CPUReq: testPod3CPUQuantity.String(), CPULim: testPod3CPUQuantity.String()},
+				Resources: &cgroups.ContainerResources{CPUReq: testPod3CPUQuantity.String(), CPULim: testPod3CPUQuantity.String()},
 			},
 		}
 		patchTestpod1ToMakeSpaceForPod3 := fmt.Sprintf(`{
@@ -413,7 +414,7 @@ func doPodResizeSchedulerTests(f *framework.Framework) {
 		expected := []podresize.ResizableContainerInfo{
 			{
 				Name:         "c1",
-				Resources:    &podresize.ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()},
+				Resources:    &cgroups.ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()},
 				RestartCount: testPod1.Status.ContainerStatuses[0].RestartCount,
 			},
 		}