diff --git a/test/e2e/common/node/framework/cgroups/cgroups.go b/test/e2e/common/node/framework/cgroups/cgroups.go index 2e9ae7bc6f7..00573c9f920 100644 --- a/test/e2e/common/node/framework/cgroups/cgroups.go +++ b/test/e2e/common/node/framework/cgroups/cgroups.go @@ -344,7 +344,7 @@ func verifyPodMemoryLimit(ctx context.Context, f *framework.Framework, pod *v1.P func VerifyPodCgroups(ctx context.Context, f *framework.Framework, pod *v1.Pod, info *ContainerResources) error { ginkgo.GinkgoHelper() - onCgroupV2 := IsPodOnCgroupv2Node(f, pod) + onCgroupV2 := IsPodOnCgroupv2Node(f, pod.Name, pod.Spec.Containers[0].Name) // Verify cgroup values expectedResources := info.ResourceRequirements() @@ -413,7 +413,7 @@ func VerifyOomScoreAdjValue(f *framework.Framework, pod *v1.Pod, cName, expected // IsPodOnCgroupv2Node checks whether the pod is running on cgroupv2 node. // TODO: Deduplicate this function with NPD cluster e2e test: // https://github.com/kubernetes/kubernetes/blob/2049360379bcc5d6467769cef112e6e492d3d2f0/test/e2e/node/node_problem_detector.go#L369 -func IsPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) (result bool) { +func IsPodOnCgroupv2Node(f *framework.Framework, podName, containerName string) (result bool) { podOnCgroupv2NodeMutex.Lock() defer podOnCgroupv2NodeMutex.Unlock() if podOnCgroupv2Node != nil { @@ -424,7 +424,7 @@ func IsPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) (result bool) { }() cmd := "mount -t cgroup2" - out, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd) + out, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, podName, containerName, "/bin/sh", "-c", cmd) if err != nil { return false } diff --git a/test/e2e/common/node/framework/podresize/resize.go b/test/e2e/common/node/framework/podresize/resize.go index 0ccee72ea16..a9426587bac 100644 --- a/test/e2e/common/node/framework/podresize/resize.go +++ b/test/e2e/common/node/framework/podresize/resize.go @@ -186,10 +186,10 @@ func VerifyPodStatusResources(gotPod *v1.Pod, wantInfo []ResizableContainerInfo) wantInitCtrs, wantCtrs := separateContainers(wantInfo) var errs []error - if err := verifyPodContainersStatusResources(gotPod.Status.InitContainerStatuses, wantInitCtrs); err != nil { + if err := VerifyPodContainersStatusResources(gotPod.Status.InitContainerStatuses, wantInitCtrs); err != nil { errs = append(errs, err) } - if err := verifyPodContainersStatusResources(gotPod.Status.ContainerStatuses, wantCtrs); err != nil { + if err := VerifyPodContainersStatusResources(gotPod.Status.ContainerStatuses, wantCtrs); err != nil { errs = append(errs, err) } @@ -208,7 +208,7 @@ func VerifyPodLevelStatusResources(gotPod *v1.Pod, wantPodResources *v1.Resource return utilerrors.NewAggregate(errs) } -func verifyPodContainersStatusResources(gotCtrStatuses []v1.ContainerStatus, wantCtrs []v1.Container) error { +func VerifyPodContainersStatusResources(gotCtrStatuses []v1.ContainerStatus, wantCtrs []v1.Container) error { ginkgo.GinkgoHelper() var errs []error @@ -295,7 +295,7 @@ func addResourceList(des, src v1.ResourceList) { func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework, pod *v1.Pod, tcInfo []ResizableContainerInfo) error { ginkgo.GinkgoHelper() - onCgroupv2 := cgroups.IsPodOnCgroupv2Node(f, pod) + onCgroupv2 := cgroups.IsPodOnCgroupv2Node(f, pod.Name, pod.Spec.Containers[0].Name) var errs []error for _, ci := range tcInfo { diff --git a/test/e2e/common/node/pod_level_resources.go b/test/e2e/common/node/pod_level_resources.go index 743fa7a1c7f..21d1ec3e8f9 100644 --- a/test/e2e/common/node/pod_level_resources.go +++ b/test/e2e/common/node/pod_level_resources.go @@ -90,7 +90,7 @@ func isCgroupv2Node(f *framework.Framework, ctx context.Context) bool { framework.ExpectNoError(delErr, "failed to delete pod %s", delErr) }() - return cgroups.IsPodOnCgroupv2Node(f, pod) + return cgroups.IsPodOnCgroupv2Node(f, pod.Name, pod.Spec.Containers[0].Name) } func makeObjectMetadata(name, namespace string) metav1.ObjectMeta { diff --git a/test/e2e/common/node/pod_resize.go b/test/e2e/common/node/pod_resize.go index 7bfccb1d590..8e551ba2fb8 100644 --- a/test/e2e/common/node/pod_resize.go +++ b/test/e2e/common/node/pod_resize.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/common/node/framework/cgroups" "k8s.io/kubernetes/test/e2e/common/node/framework/podresize" "k8s.io/kubernetes/test/e2e/feature" @@ -576,34 +577,6 @@ func doPodResizePatchErrorTests(f *framework.Framework) { "only cpu and memory resources are mutable", true, ), - ginkgo.Entry("Burstable pod - nonrestartable initContainer", - []podresize.ResizableContainerInfo{ - { - Name: "c1-init", - InitCtr: true, - Resources: &cgroups.ContainerResources{CPUReq: originalCPU, MemReq: originalMem}, - RestartPolicy: v1.ContainerRestartPolicyNever, - }, - { - Name: "c1", - Resources: &cgroups.ContainerResources{}, - }, - }, - []podresize.ResizableContainerInfo{ - { - Name: "c1-init", - InitCtr: true, - Resources: &cgroups.ContainerResources{CPUReq: increasedCPU, MemReq: increasedMem}, - RestartPolicy: v1.ContainerRestartPolicyNever, - }, - { - Name: "c1", - Resources: &cgroups.ContainerResources{}, - }, - }, - "resources for non-sidecar init containers are immutable", - false, - ), ) } @@ -718,6 +691,86 @@ func doPodResizeMemoryLimitDecreaseTest(f *framework.Framework) { }) } +func doPodResizeInitContainerResizeTest(f *framework.Framework) { + ginkgo.It("should support resizing a non-sidecar init container", func(ctx context.Context) { + // A command that never exits, to allow us to patch the init container while it's running. + cmd := "tail -f /dev/null" + + originalContainers := []podresize.ResizableContainerInfo{ + { + Name: "init-c1", + InitCtr: true, + Resources: &cgroups.ContainerResources{CPUReq: originalCPU, CPULim: originalCPU, MemReq: originalMem, MemLim: originalMem}, + RestartPolicy: v1.ContainerRestartPolicyNever, + }, + { + Name: "main-c1", + Resources: &cgroups.ContainerResources{CPUReq: originalCPU, CPULim: originalCPU, MemReq: originalMem, MemLim: originalMem}, + }, + } + resizedContainers := []podresize.ResizableContainerInfo{ + { + Name: "init-c1", + InitCtr: true, + Resources: &cgroups.ContainerResources{CPUReq: increasedCPU, CPULim: increasedCPU, MemReq: increasedMem, MemLim: increasedMem}, + RestartPolicy: v1.ContainerRestartPolicyNever, + }, + { + Name: "main-c1", + Resources: originalContainers[1].Resources, + }, + } + + originalInitCtr := cgroups.MakeContainerWithResources( + originalContainers[0].Name, + originalContainers[0].Resources, + cmd, + ) + resizedInitCtr := cgroups.MakeContainerWithResources( + resizedContainers[0].Name, + resizedContainers[0].Resources, + cmd, + ) + + ginkgo.By("Creating pod with an nonrestartable init container") + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "resize-test-", + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{originalInitCtr}, + Containers: []v1.Container{cgroups.MakeContainerWithResources( + originalContainers[1].Name, + originalContainers[1].Resources, + cmd, + )}, + }, + } + testPod = e2epod.MustMixinRestrictedPodSecurity(testPod) + podClient := e2epod.NewPodClient(f) + newPod := podClient.Create(ctx, testPod) + verifyInitContainerResources(ctx, f, newPod, originalContainers, originalInitCtr) + + ginkgo.By("patching and verifying pod for resize") + patch := podresize.MakeResizePatch(originalContainers, resizedContainers, nil, nil) + patchedPod, pErr := f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(ctx, newPod.Name, + types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "resize") + framework.ExpectNoError(pErr, "failed to patch pod for resize") + verifyInitContainerResources(ctx, f, patchedPod, resizedContainers, resizedInitCtr) + + // Resize has been actuated, test the reverse operation. + ginkgo.By("patching and verifying pod to revert to original resources") + patch = podresize.MakeResizePatch(resizedContainers, originalContainers, nil, nil) + patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(ctx, newPod.Name, + types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "resize") + framework.ExpectNoError(pErr, "failed to patch pod for resize") + verifyInitContainerResources(ctx, f, patchedPod, originalContainers, originalInitCtr) + + ginkgo.By("deleting pod") + podClient.DeleteSync(ctx, newPod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete) + }) +} + func doPodResizeReadAndReplaceTests(f *framework.Framework) { /* Release: v1.35 @@ -826,6 +879,20 @@ var _ = SIGDescribe("Pod InPlace Resize Container", func() { doPodResizeMemoryLimitDecreaseTest(f) }) +var _ = SIGDescribe("Pod InPlace Resize Init Container", framework.WithFeatureGate(features.InPlacePodVerticalScalingInitContainers), func() { + f := framework.NewDefaultFramework("pod-resize-tests") + + ginkgo.BeforeEach(func(ctx context.Context) { + _, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + if framework.NodeOSDistroIs("windows") { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + }) + + doPodResizeInitContainerResizeTest(f) +}) + func doPatchAndRollback(ctx context.Context, f *framework.Framework, originalContainers, expectedContainers []podresize.ResizableContainerInfo, originalPodResources, expectedPodResources *v1.ResourceRequirements, doRollback bool, mountPodCgroup bool) { ginkgo.By("creating and verifying pod") podClient := e2epod.NewPodClient(f) @@ -875,8 +942,8 @@ func createAndVerifyPod(ctx context.Context, f *framework.Framework, podClient * } newPod := podClient.CreateSync(ctx, testPod) - podresize.VerifyPodResources(newPod, originalContainers, podResources) + podresize.VerifyPodResources(newPod, originalContainers, podResources) podresize.VerifyPodResizePolicy(newPod, originalContainers) framework.ExpectNoError(podresize.VerifyPodStatusResources(newPod, originalContainers)) framework.ExpectNoError(podresize.VerifyPodContainersCgroupValues(ctx, f, newPod, originalContainers)) @@ -957,3 +1024,34 @@ func createRollbackContainers(originalContainers, expectedContainers []podresize } return rollbackContainers } + +func verifyInitContainerResources(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectedContainers []podresize.ResizableContainerInfo, expectedInitCtr v1.Container) { + ginkgo.GinkgoHelper() + + podresize.VerifyPodResizePolicy(pod, expectedContainers) + podresize.VerifyPodResources(pod, expectedContainers, nil) + + // Wait for the init container to display the expected resources in its status, and that + // the cgroups are as expected. + framework.ExpectNoError(framework.Gomega(). + Eventually(ctx, framework.RetryNotFound(framework.GetObject(f.ClientSet.CoreV1().Pods(pod.Namespace).Get, pod.Name, metav1.GetOptions{}))). + WithTimeout(f.Timeouts.PodStart). + Should(framework.MakeMatcher(func(pod *v1.Pod) (func() string, error) { + if pod == nil || len(pod.Status.InitContainerStatuses) == 0 || pod.Status.InitContainerStatuses[0].Resources == nil { + return func() string { return "pod or init container status is not available yet" }, nil + } + if err := podresize.VerifyPodContainersStatusResources(pod.Status.InitContainerStatuses, []v1.Container{expectedInitCtr}); err != nil { + return func() string { + return fmt.Sprintf("init container status resources don't match expected: %v", err) + }, nil + } + onCgroupv2 := cgroups.IsPodOnCgroupv2Node(f, pod.Name, pod.Spec.InitContainers[0].Name) + if err := cgroups.VerifyContainerCgroupValues(ctx, f, pod, &expectedInitCtr, onCgroupv2); err != nil { + return func() string { + return fmt.Sprintf("cgroup values for init container don't match expected: %v", err) + }, nil + } + return nil, nil + })), + ) +}