From 99a6153a4f39d0248e21fad5c1007d2ce9e4ee6c Mon Sep 17 00:00:00 2001 From: ndixita Date: Sun, 3 Nov 2024 22:01:48 +0000 Subject: [PATCH] e2e tests Signed-off-by: ndixita --- test/e2e/common/node/pod_level_resources.go | 408 ++++++++++++++++++++ test/e2e/feature/feature.go | 5 + test/e2e/framework/pod/resize.go | 36 +- test/e2e/framework/pod/utils.go | 32 ++ test/utils/node.go | 2 +- 5 files changed, 451 insertions(+), 32 deletions(-) create mode 100644 test/e2e/common/node/pod_level_resources.go diff --git a/test/e2e/common/node/pod_level_resources.go b/test/e2e/common/node/pod_level_resources.go new file mode 100644 index 00000000000..8dbe18b1d8b --- /dev/null +++ b/test/e2e/common/node/pod_level_resources.go @@ -0,0 +1,408 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + kubecm "k8s.io/kubernetes/pkg/kubelet/cm" + "k8s.io/kubernetes/test/e2e/feature" + "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + imageutils "k8s.io/kubernetes/test/utils/image" + admissionapi "k8s.io/pod-security-admission/api" +) + +const ( + cgroupv2CPUWeight string = "cpu.weight" + cgroupv2CPULimit string = "cpu.max" + cgroupv2MemLimit string = "memory.max" + cgroupFsPath string = "/sys/fs/cgroup" + CPUPeriod string = "100000" + mountPath string = "/sysfscgroup" +) + +var ( + cmd = []string{"/bin/sh", "-c", "sleep 1d"} +) + +var _ = SIGDescribe("Pod Level Resources", framework.WithSerial(), feature.PodLevelResources, "[NodeAlphaFeature:PodLevelResources]", func() { + f := framework.NewDefaultFramework("pod-level-resources-tests") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + ginkgo.BeforeEach(func(ctx context.Context) { + _, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + + if framework.NodeOSDistroIs("windows") { + e2eskipper.Skipf("not supported on windows -- skipping") + } + + // skip the test on nodes with cgroupv2 not enabled. + if !isCgroupv2Node(f, ctx) { + e2eskipper.Skipf("not supported on cgroupv1 -- skipping") + } + }) + podLevelResourcesTests(f) +}) + +// isCgroupv2Node creates a small pod and check if it is running on a node +// with cgroupv2 enabled. +// TODO: refactor to mark this test with cgroupv2 label, and rather check +// the label in the test job, to tun this test on a node with cgroupv2. +func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool { + podClient := e2epod.NewPodClient(f) + cgroupv2Testpod := &v1.Pod{ + ObjectMeta: makeObjectMetadata("cgroupv2-check", f.Namespace.Name), + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "cgroupv2-check", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: cmd, + Resources: getResourceRequirements(&resourceInfo{CPULim: "1m", MemReq: "1Mi"}), + }, + }, + }, + } + + pod := podClient.CreateSync(ctx, cgroupv2Testpod) + defer func() { + framework.Logf("Deleting %q pod", cgroupv2Testpod.Name) + delErr := e2epod.DeletePodWithWait(ctx, f.ClientSet, pod) + framework.ExpectNoError(delErr, "failed to delete pod %s", delErr) + }() + + return e2epod.IsPodOnCgroupv2Node(f, pod) +} + +func makeObjectMetadata(name, namespace string) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: "testpod", Namespace: namespace, + Labels: map[string]string{"time": strconv.Itoa(time.Now().Nanosecond())}, + } +} + +type containerInfo struct { + Name string + Resources *resourceInfo +} +type resourceInfo struct { + CPUReq string + CPULim string + MemReq string + MemLim string +} + +func makeContainer(info containerInfo) v1.Container { + cmd := []string{"/bin/sh", "-c", "sleep 1d"} + res := getResourceRequirements(info.Resources) + return v1.Container{ + Name: info.Name, + Command: cmd, + Resources: res, + Image: imageutils.GetE2EImage(imageutils.BusyBox), + VolumeMounts: []v1.VolumeMount{ + { + Name: "sysfscgroup", + MountPath: mountPath, + }, + }, + } +} + +func getResourceRequirements(info *resourceInfo) v1.ResourceRequirements { + var res v1.ResourceRequirements + if info != nil { + if info.CPUReq != "" || info.MemReq != "" { + res.Requests = make(v1.ResourceList) + } + if info.CPUReq != "" { + res.Requests[v1.ResourceCPU] = resource.MustParse(info.CPUReq) + } + if info.MemReq != "" { + res.Requests[v1.ResourceMemory] = resource.MustParse(info.MemReq) + } + + if info.CPULim != "" || info.MemLim != "" { + res.Limits = make(v1.ResourceList) + } + if info.CPULim != "" { + res.Limits[v1.ResourceCPU] = resource.MustParse(info.CPULim) + } + if info.MemLim != "" { + res.Limits[v1.ResourceMemory] = resource.MustParse(info.MemLim) + } + } + return res +} + +func makePod(metadata *metav1.ObjectMeta, podResources *resourceInfo, containers []containerInfo) *v1.Pod { + var testContainers []v1.Container + for _, container := range containers { + testContainers = append(testContainers, makeContainer(container)) + } + + pod := &v1.Pod{ + ObjectMeta: *metadata, + + Spec: v1.PodSpec{ + Containers: testContainers, + Volumes: []v1.Volume{ + { + Name: "sysfscgroup", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: cgroupFsPath}, + }, + }, + }, + }, + } + + if podResources != nil { + res := getResourceRequirements(podResources) + pod.Spec.Resources = &res + } + + return pod +} + +func verifyPodResources(gotPod v1.Pod, inputInfo, expectedInfo *resourceInfo) { + ginkgo.GinkgoHelper() + var expectedResources *v1.ResourceRequirements + // expectedResources will be nil if pod-level resources are not set in the test + // case input. + if inputInfo != nil { + resourceInfo := getResourceRequirements(expectedInfo) + expectedResources = &resourceInfo + } + gomega.Expect(expectedResources).To(gomega.Equal(gotPod.Spec.Resources)) +} + +func verifyQoS(gotPod v1.Pod, expectedQoS v1.PodQOSClass) { + ginkgo.GinkgoHelper() + gomega.Expect(expectedQoS).To(gomega.Equal(gotPod.Status.QOSClass)) +} + +// TODO(ndixita): dedup the conversion logic in pod resize test and move to helpers/utils. +func verifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, info *resourceInfo) error { + ginkgo.GinkgoHelper() + cmd := fmt.Sprintf("find %s -name '*%s*'", mountPath, strings.ReplaceAll(string(pod.UID), "-", "_")) + framework.Logf("Namespace %s Pod %s - looking for Pod cgroup directory path: %q", f.Namespace, pod.Name, cmd) + podCgPath, stderr, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, []string{"/bin/sh", "-c", cmd}...) + if err != nil || len(stderr) > 0 { + return fmt.Errorf("encountered error while running command: %q, \nerr: %w \nstdErr: %q", cmd, err, stderr) + } + + expectedResources := getResourceRequirements(info) + cpuWeightCgPath := fmt.Sprintf("%s/%s", podCgPath, cgroupv2CPUWeight) + expectedCPUShares := int64(kubecm.MilliCPUToShares(expectedResources.Requests.Cpu().MilliValue())) + expectedCPUShares = int64(1 + ((expectedCPUShares-2)*9999)/262142) + // convert cgroup v1 cpu.shares value to cgroup v2 cpu.weight value + // https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2 + var errs []error + err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuWeightCgPath, strconv.FormatInt(expectedCPUShares, 10)) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify cpu request cgroup value: %w", err)) + } + + cpuLimCgPath := fmt.Sprintf("%s/%s", podCgPath, cgroupv2CPULimit) + cpuQuota := kubecm.MilliCPUToQuota(expectedResources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod) + expectedCPULimit := strconv.FormatInt(cpuQuota, 10) + expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod) + err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, cpuLimCgPath, expectedCPULimit) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)) + } + + memLimCgPath := fmt.Sprintf("%s/%s", podCgPath, cgroupv2MemLimit) + expectedMemLim := strconv.FormatInt(expectedResources.Limits.Memory().Value(), 10) + err = e2epod.VerifyCgroupValue(f, pod, pod.Spec.Containers[0].Name, memLimCgPath, expectedMemLim) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) + } + return utilerrors.NewAggregate(errs) +} + +func podLevelResourcesTests(f *framework.Framework) { + type expectedPodConfig struct { + qos v1.PodQOSClass + // totalPodResources represents the aggregate resource requests + // and limits for the pod. If pod-level resource specifications + // are specified, totalPodResources is equal to pod-level resources. + // Otherwise, it is calculated by aggregating resource requests and + // limits from all containers within the pod.. + totalPodResources *resourceInfo + } + + type testCase struct { + name string + podResources *resourceInfo + containers []containerInfo + expected expectedPodConfig + } + + tests := []testCase{ + { + name: "Guaranteed QoS pod with container resources", + containers: []containerInfo{ + {Name: "c1", Resources: &resourceInfo{CPUReq: "50m", CPULim: "50m", MemReq: "70Mi", MemLim: "70Mi"}}, + {Name: "c2", Resources: &resourceInfo{CPUReq: "70m", CPULim: "70m", MemReq: "50Mi", MemLim: "50Mi"}}, + }, + expected: expectedPodConfig{ + qos: v1.PodQOSGuaranteed, + totalPodResources: &resourceInfo{CPUReq: "120m", CPULim: "120m", MemReq: "120Mi", MemLim: "120Mi"}, + }, + }, + { + name: "Guaranteed QoS pod, no container resources", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + containers: []containerInfo{{Name: "c1"}, {Name: "c2"}}, + expected: expectedPodConfig{ + qos: v1.PodQOSGuaranteed, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + }, + }, + { + name: "Guaranteed QoS pod with container resources", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + containers: []containerInfo{ + {Name: "c1", Resources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}}, + {Name: "c2", Resources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}}, + }, + expected: expectedPodConfig{ + qos: v1.PodQOSGuaranteed, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + }, + }, + { + name: "Guaranteed QoS pod, 1 container with resources", + podResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + containers: []containerInfo{ + {Name: "c1", Resources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}}, + {Name: "c2"}, + }, + expected: expectedPodConfig{ + qos: v1.PodQOSGuaranteed, + totalPodResources: &resourceInfo{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + }, + }, + { + name: "Burstable QoS pod, no container resources", + podResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, + containers: []containerInfo{ + {Name: "c1"}, + {Name: "c2"}, + }, + expected: expectedPodConfig{ + qos: v1.PodQOSBurstable, + totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, + }, + }, + { + name: "Burstable QoS pod with container resources", + podResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, + containers: []containerInfo{ + {Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "100m", MemReq: "20Mi", MemLim: "100Mi"}}, + {Name: "c2", Resources: &resourceInfo{CPUReq: "30m", CPULim: "100m", MemReq: "30Mi", MemLim: "100Mi"}}, + }, + expected: expectedPodConfig{ + qos: v1.PodQOSBurstable, + totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, + }, + }, + { + name: "Burstable QoS pod, 1 container with resources", + podResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, + containers: []containerInfo{ + {Name: "c1", Resources: &resourceInfo{CPUReq: "20m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}}, + {Name: "c2"}, + }, + expected: expectedPodConfig{ + qos: v1.PodQOSBurstable, + totalPodResources: &resourceInfo{CPUReq: "50m", CPULim: "100m", MemReq: "50Mi", MemLim: "100Mi"}, + }, + }, + } + + for _, tc := range tests { + ginkgo.It(tc.name, func(ctx context.Context) { + podMetadata := makeObjectMetadata("testpod", f.Namespace.Name) + testPod := makePod(&podMetadata, tc.podResources, tc.containers) + + ginkgo.By("creating pods") + podClient := e2epod.NewPodClient(f) + pod := podClient.CreateSync(ctx, testPod) + + ginkgo.By("verifying pod resources are as expected") + verifyPodResources(*pod, tc.podResources, tc.expected.totalPodResources) + + ginkgo.By("verifying pod QoS as expected") + verifyQoS(*pod, tc.expected.qos) + + ginkgo.By("verifying pod cgroup values") + err := verifyPodCgroups(ctx, f, pod, tc.expected.totalPodResources) + framework.ExpectNoError(err, "failed to verify pod's cgroup values: %v", err) + + ginkgo.By("verifying containers cgroup limits are same as pod container's cgroup limits") + err = verifyContainersCgroupLimits(f, pod) + framework.ExpectNoError(err, "failed to verify containers cgroup values: %v", err) + + ginkgo.By("deleting pods") + delErr := e2epod.DeletePodWithWait(ctx, f.ClientSet, pod) + framework.ExpectNoError(delErr, "failed to delete pod %s", delErr) + }) + } +} + +func verifyContainersCgroupLimits(f *framework.Framework, pod *v1.Pod) error { + var errs []error + for _, container := range pod.Spec.Containers { + if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Memory() != nil && + container.Resources.Limits.Memory() == nil { + expectedCgroupMemLimit := strconv.FormatInt(pod.Spec.Resources.Limits.Memory().Value(), 10) + err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2MemLimit), expectedCgroupMemLimit) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify memory limit cgroup value: %w", err)) + } + } + + if pod.Spec.Resources != nil && pod.Spec.Resources.Limits.Cpu() != nil && + container.Resources.Limits.Cpu() == nil { + cpuQuota := kubecm.MilliCPUToQuota(pod.Spec.Resources.Limits.Cpu().MilliValue(), kubecm.QuotaPeriod) + expectedCPULimit := strconv.FormatInt(cpuQuota, 10) + expectedCPULimit = fmt.Sprintf("%s %s", expectedCPULimit, CPUPeriod) + err := e2epod.VerifyCgroupValue(f, pod, container.Name, fmt.Sprintf("%s/%s", cgroupFsPath, cgroupv2CPULimit), expectedCPULimit) + if err != nil { + errs = append(errs, fmt.Errorf("failed to verify cpu limit cgroup value: %w", err)) + } + } + } + return utilerrors.NewAggregate(errs) +} diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go index d703a15da48..d58ab30da3f 100644 --- a/test/e2e/feature/feature.go +++ b/test/e2e/feature/feature.go @@ -268,6 +268,11 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) PodGarbageCollector = framework.WithFeature(framework.ValidFeatures.Add("PodGarbageCollector")) + // owner: sig-node + // Marks a test for for pod-level resources feature that requires + // PodLevelResources feature gate to be enabled. + PodLevelResources = framework.WithFeature(framework.ValidFeatures.Add("PodLevelResources")) + // TODO: document the feature (owning SIG, when to use this feature for a test) PodLifecycleSleepAction = framework.WithFeature(framework.ValidFeatures.Add("PodLifecycleSleepAction")) diff --git a/test/e2e/framework/pod/resize.go b/test/e2e/framework/pod/resize.go index 2e189d02ae7..bc01b346450 100644 --- a/test/e2e/framework/pod/resize.go +++ b/test/e2e/framework/pod/resize.go @@ -243,22 +243,10 @@ func VerifyPodStatusResources(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) return utilerrors.NewAggregate(errs) } -// isPodOnCgroupv2Node checks whether the pod is running on cgroupv2 node. -// TODO: Deduplicate this function with NPD cluster e2e test: -// https://github.com/kubernetes/kubernetes/blob/2049360379bcc5d6467769cef112e6e492d3d2f0/test/e2e/node/node_problem_detector.go#L369 -func isPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) bool { - cmd := "mount -t cgroup2" - out, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd) - if err != nil { - return false - } - return len(out) != 0 -} - func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework, pod *v1.Pod, tcInfo []ResizableContainerInfo) error { ginkgo.GinkgoHelper() if podOnCgroupv2Node == nil { - value := isPodOnCgroupv2Node(f, pod) + value := IsPodOnCgroupv2Node(f, pod) podOnCgroupv2Node = &value } cgroupMemLimit := Cgroupv2MemLimit @@ -269,21 +257,7 @@ func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework cgroupCPULimit = CgroupCPUQuota cgroupCPURequest = CgroupCPUShares } - verifyCgroupValue := func(cName, cgPath, expectedCgValue string) error { - cmd := fmt.Sprintf("head -n 1 %s", cgPath) - framework.Logf("Namespace %s Pod %s Container %s - looking for cgroup value %s in path %s", - pod.Namespace, pod.Name, cName, expectedCgValue, cgPath) - cgValue, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd) - if err != nil { - return fmt.Errorf("failed to read cgroup %q for container %s: %w", cgPath, cName, err) - } - cgValue = strings.Trim(cgValue, "\n") - if cgValue != expectedCgValue { - return fmt.Errorf("container %s cgroup %q doesn't match expected: got %q want %q", - cName, cgPath, cgValue, expectedCgValue) - } - return nil - } + var errs []error for _, ci := range tcInfo { if ci.Resources == nil { @@ -320,10 +294,10 @@ func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework expectedCPUShares = int64(1 + ((expectedCPUShares-2)*9999)/262142) } if expectedMemLimitString != "0" { - errs = append(errs, verifyCgroupValue(ci.Name, cgroupMemLimit, expectedMemLimitString)) + errs = append(errs, VerifyCgroupValue(f, pod, ci.Name, cgroupMemLimit, expectedMemLimitString)) } - errs = append(errs, verifyCgroupValue(ci.Name, cgroupCPULimit, expectedCPULimitString)) - errs = append(errs, verifyCgroupValue(ci.Name, cgroupCPURequest, strconv.FormatInt(expectedCPUShares, 10))) + errs = append(errs, VerifyCgroupValue(f, pod, ci.Name, cgroupCPULimit, expectedCPULimitString)) + errs = append(errs, VerifyCgroupValue(f, pod, ci.Name, cgroupCPURequest, strconv.FormatInt(expectedCPUShares, 10))) } } return utilerrors.NewAggregate(errs) diff --git a/test/e2e/framework/pod/utils.go b/test/e2e/framework/pod/utils.go index cf33d9f1771..d28a26c3311 100644 --- a/test/e2e/framework/pod/utils.go +++ b/test/e2e/framework/pod/utils.go @@ -19,11 +19,13 @@ package pod import ( "flag" "fmt" + "strings" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/test/e2e/framework" imageutils "k8s.io/kubernetes/test/utils/image" psaapi "k8s.io/pod-security-admission/api" psapolicy "k8s.io/pod-security-admission/policy" @@ -275,3 +277,33 @@ func FindContainerStatusInPod(pod *v1.Pod, containerName string) *v1.ContainerSt } return nil } + +// VerifyCgroupValue verifies that the given cgroup path has the expected value in +// the specified container of the pod. It execs into the container to retrive the +// cgroup value and compares it against the expected value. +func VerifyCgroupValue(f *framework.Framework, pod *v1.Pod, cName, cgPath, expectedCgValue string) error { + cmd := fmt.Sprintf("head -n 1 %s", cgPath) + framework.Logf("Namespace %s Pod %s Container %s - looking for cgroup value %s in path %s", + pod.Namespace, pod.Name, cName, expectedCgValue, cgPath) + cgValue, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd) + if err != nil { + return fmt.Errorf("failed to find expected value %q in container cgroup %q", expectedCgValue, cgPath) + } + cgValue = strings.Trim(cgValue, "\n") + if cgValue != expectedCgValue { + return fmt.Errorf("cgroup value %q not equal to expected %q", cgValue, expectedCgValue) + } + return nil +} + +// IsPodOnCgroupv2Node checks whether the pod is running on cgroupv2 node. +// TODO: Deduplicate this function with NPD cluster e2e test: +// https://github.com/kubernetes/kubernetes/blob/2049360379bcc5d6467769cef112e6e492d3d2f0/test/e2e/node/node_problem_detector.go#L369 +func IsPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) bool { + cmd := "mount -t cgroup2" + out, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd) + if err != nil { + return false + } + return len(out) != 0 +} diff --git a/test/utils/node.go b/test/utils/node.go index 9388251acd0..3483088aa72 100644 --- a/test/utils/node.go +++ b/test/utils/node.go @@ -16,7 +16,7 @@ limitations under the License. package utils -import "k8s.io/api/core/v1" +import v1 "k8s.io/api/core/v1" // GetNodeCondition extracts the provided condition from the given status and returns that. // Returns nil and -1 if the condition is not present, and the index of the located condition.