From 6203006348e25b6edf06d550de3bbccf002495f5 Mon Sep 17 00:00:00 2001 From: Anish Shah Date: Thu, 17 Oct 2024 05:09:56 -0700 Subject: [PATCH 1/4] test: parity between cluster and node IPPR e2e tests Some IPPR cluster e2e tests are missing from node e2e tests. This change brings parity between them. --- test/e2e_node/pod_resize_test.go | 256 +++++++++++++++++++++++++------ 1 file changed, 211 insertions(+), 45 deletions(-) diff --git a/test/e2e_node/pod_resize_test.go b/test/e2e_node/pod_resize_test.go index 61c8a741140..56f7f35315a 100644 --- a/test/e2e_node/pod_resize_test.go +++ b/test/e2e_node/pod_resize_test.go @@ -54,6 +54,8 @@ const ( Cgroupv2CPURequest string = "/sys/fs/cgroup/cpu.weight" CPUPeriod string = "100000" MinContainerRuntimeVersion string = "1.6.9" + + fakeExtendedResource = "dummy.com/dummy" ) var ( @@ -64,18 +66,21 @@ var ( ) type ContainerResources struct { - CPUReq string - CPULim string - MemReq string - MemLim string - EphStorReq string - EphStorLim string + CPUReq string + CPULim string + MemReq string + MemLim string + EphStorReq string + EphStorLim string + ExtendedResourceReq string + ExtendedResourceLim string } type ContainerAllocations struct { - CPUAlloc string - MemAlloc string - ephStorAlloc string + CPUAlloc string + MemAlloc string + ephStorAlloc string + ExtendedResourceAlloc string } type TestContainerInfo struct { @@ -87,6 +92,28 @@ type TestContainerInfo struct { RestartCount int32 } +type containerPatch struct { + Name string `json:"name"` + Resources struct { + Requests struct { + CPU string `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` + EphStor string `json:"ephemeral-storage,omitempty"` + } `json:"requests"` + Limits struct { + CPU string `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` + EphStor string `json:"ephemeral-storage,omitempty"` + } `json:"limits"` + } `json:"resources"` +} + +type patchSpec struct { + Spec struct { + Containers []containerPatch `json:"containers"` + } `json:"spec"` +} + func supportsInPlacePodVerticalScaling(ctx context.Context, f *framework.Framework) bool { node := getLocalNode(ctx, f) re := regexp.MustCompile("containerd://(.*)") @@ -418,6 +445,100 @@ func waitForPodResizeActuation(ctx context.Context, f *framework.Framework, c cl return resizedPod } +func genPatchString(containers []TestContainerInfo) (string, error) { + var patch patchSpec + + for _, container := range containers { + var cPatch containerPatch + cPatch.Name = container.Name + cPatch.Resources.Requests.CPU = container.Resources.CPUReq + cPatch.Resources.Requests.Memory = container.Resources.MemReq + cPatch.Resources.Limits.CPU = container.Resources.CPULim + cPatch.Resources.Limits.Memory = container.Resources.MemLim + + patch.Spec.Containers = append(patch.Spec.Containers, cPatch) + } + + patchBytes, err := json.Marshal(patch) + if err != nil { + return "", err + } + + return string(patchBytes), nil +} + +func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error { + oldData, err := json.Marshal(old) + if err != nil { + return err + } + + newData, err := json.Marshal(new) + if err != nil { + return err + } + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{}) + if err != nil { + return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err) + } + _, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") + return err +} + +func addExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string, extendedResourceQuantity resource.Quantity) { + extendedResource := v1.ResourceName(extendedResourceName) + + ginkgo.By("Adding a custom resource") + OriginalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + node := OriginalNode.DeepCopy() + node.Status.Capacity[extendedResource] = extendedResourceQuantity + node.Status.Allocatable[extendedResource] = extendedResourceQuantity + err = patchNode(context.Background(), clientSet, OriginalNode.DeepCopy(), node) + framework.ExpectNoError(err) + + gomega.Eventually(func() error { + node, err = clientSet.CoreV1().Nodes().Get(context.Background(), node.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + fakeResourceCapacity, exists := node.Status.Capacity[extendedResource] + if !exists { + return fmt.Errorf("node %s has no %s resource capacity", node.Name, extendedResourceName) + } + if expectedResource := resource.MustParse("123"); fakeResourceCapacity.Cmp(expectedResource) != 0 { + return fmt.Errorf("node %s has resource capacity %s, expected: %s", node.Name, fakeResourceCapacity.String(), expectedResource.String()) + } + + return nil + }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) +} + +func removeExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string) { + extendedResource := v1.ResourceName(extendedResourceName) + + ginkgo.By("Removing a custom resource") + originalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + node := originalNode.DeepCopy() + delete(node.Status.Capacity, extendedResource) + delete(node.Status.Allocatable, extendedResource) + err = patchNode(context.Background(), clientSet, originalNode.DeepCopy(), node) + framework.ExpectNoError(err) + + gomega.Eventually(func() error { + node, err = clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + if _, exists := node.Status.Capacity[extendedResource]; exists { + return fmt.Errorf("node %s has resource capacity %s which is expected to be removed", node.Name, extendedResourceName) + } + + return nil + }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) +} + func doPodResizeTests() { f := framework.NewDefaultFramework("pod-resize-test") var podClient *e2epod.PodClient @@ -426,10 +547,11 @@ func doPodResizeTests() { }) type testCase struct { - name string - containers []TestContainerInfo - patchString string - expected []TestContainerInfo + name string + containers []TestContainerInfo + patchString string + expected []TestContainerInfo + addExtendedResource bool } noRestart := v1.NotRequired @@ -1131,6 +1253,31 @@ func doPodResizeTests() { }, }, }, + { + name: "Guaranteed QoS pod, one container - increase CPU & memory with an extended resource", + containers: []TestContainerInfo{ + { + Name: "c1", + Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi", + ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} + ]}}`, + expected: []TestContainerInfo{ + { + Name: "c1", + Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi", + ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + addExtendedResource: true, + }, } timeouts := framework.NewTimeoutContext() @@ -1153,6 +1300,20 @@ func doPodResizeTests() { testPod = makeTestPod(f.Namespace.Name, "testpod", tStamp, tc.containers) testPod = e2epod.MustMixinRestrictedPodSecurity(testPod) + if tc.addExtendedResource { + nodes, err := e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + + for _, node := range nodes.Items { + addExtendedResource(f.ClientSet, node.Name, fakeExtendedResource, resource.MustParse("123")) + } + defer func() { + for _, node := range nodes.Items { + removeExtendedResource(f.ClientSet, node.Name, fakeExtendedResource) + } + }() + } + ginkgo.By("creating pod") newPod := podClient.CreateSync(ctx, testPod) @@ -1161,41 +1322,49 @@ func doPodResizeTests() { ginkgo.By("verifying initial pod resize policy is as expected") verifyPodResizePolicy(newPod, tc.containers) - err := e2epod.WaitForPodCondition(ctx, f.ClientSet, newPod.Namespace, newPod.Name, "Ready", timeouts.PodStartShort, testutils.PodRunningReady) - framework.ExpectNoError(err, "pod %s/%s did not go running", newPod.Namespace, newPod.Name) - framework.Logf("pod %s/%s running", newPod.Namespace, newPod.Name) - ginkgo.By("verifying initial pod status resources") verifyPodStatusResources(newPod, tc.containers) - ginkgo.By("patching pod for resize") - patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(ctx, newPod.Name, - types.StrategicMergePatchType, []byte(tc.patchString), metav1.PatchOptions{}) - framework.ExpectNoError(pErr, "failed to patch pod for resize") + ginkgo.By("verifying initial cgroup config are as expected") + framework.ExpectNoError(verifyPodContainersCgroupValues(ctx, f, newPod, tc.containers)) - ginkgo.By("verifying pod patched for resize") - verifyPodResources(patchedPod, tc.expected) - gomega.Eventually(ctx, verifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). - WithArguments(patchedPod, tc.containers). - Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") + patchAndVerify := func(patchString string, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, opStr string, isRollback bool) { + ginkgo.By(fmt.Sprintf("patching pod for %s", opStr)) + patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(context.TODO(), newPod.Name, + types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}) + framework.ExpectNoError(pErr, fmt.Sprintf("failed to patch pod for %s", opStr)) - ginkgo.By("waiting for resize to be actuated") - resizedPod := waitForPodResizeActuation(ctx, f, f.ClientSet, podClient, newPod, patchedPod, tc.expected) + ginkgo.By(fmt.Sprintf("verifying pod patched for %s", opStr)) + verifyPodResources(patchedPod, expectedContainers) + gomega.Eventually(ctx, verifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + WithArguments(patchedPod, initialContainers). + Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") - ginkgo.By("verifying pod resources after resize") - verifyPodResources(resizedPod, tc.expected) + ginkgo.By(fmt.Sprintf("waiting for %s to be actuated", opStr)) + resizedPod := waitForPodResizeActuation(ctx, f, podClient, newPod, patchedPod, expectedContainers, initialContainers, isRollback) - ginkgo.By("verifying pod allocations after resize") - gomega.Eventually(ctx, verifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). - WithArguments(resizedPod, tc.expected). - Should(gomega.BeNil(), "failed to verify Pod allocations for resizedPod") + // Check cgroup values only for containerd versions before 1.6.9 + ginkgo.By(fmt.Sprintf("verifying pod container's cgroup values after %s", opStr)) + framework.ExpectNoError(verifyPodContainersCgroupValues(ctx, f, resizedPod, expectedContainers)) + + ginkgo.By(fmt.Sprintf("verifying pod resources after %s", opStr)) + verifyPodResources(resizedPod, expectedContainers) + + ginkgo.By(fmt.Sprintf("verifying pod allocations after %s", opStr)) + gomega.Eventually(ctx, verifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + WithArguments(resizedPod, expectedContainers). + Should(gomega.BeNil(), "failed to verify Pod allocations for resizedPod") + } + + patchAndVerify(tc.patchString, tc.expected, tc.containers, "resize", false) + + rbPatchStr, err := genPatchString(tc.containers) + framework.ExpectNoError(err) + // Resize has been actuated, test rollback + patchAndVerify(rbPatchStr, tc.containers, tc.expected, "rollback", true) ginkgo.By("deleting pod") - deletePodSyncByName(ctx, f, newPod.Name) - // we need to wait for all containers to really be gone so cpumanager reconcile loop will not rewrite the cpu_manager_state. - // this is in turn needed because we will have an unavoidable (in the current framework) race with the - // reconcile loop which will make our attempt to delete the state file and to restore the old config go haywire - waitForAllContainerRemoval(ctx, newPod.Name, newPod.Namespace) + podClient.DeleteSync(ctx, newPod.Name, metav1.DeleteOptions{}, timeouts.PodDelete) }) } } @@ -1286,11 +1455,8 @@ func doPodResizeErrorTests() { WithArguments(patchedPod, tc.expected). Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") - deletePodSyncByName(ctx, f, newPod.Name) - // we need to wait for all containers to really be gone so cpumanager reconcile loop will not rewrite the cpu_manager_state. - // this is in turn needed because we will have an unavoidable (in the current framework) race with the - // reconcile loop which will make our attempt to delete the state file and to restore the old config go haywire - waitForAllContainerRemoval(ctx, newPod.Name, newPod.Namespace) + ginkgo.By("deleting pod") + podClient.DeleteSync(ctx, newPod.Name, metav1.DeleteOptions{}, timeouts.PodDelete) }) } } @@ -1301,7 +1467,7 @@ func doPodResizeErrorTests() { // b) api-server in services doesn't start with --enable-admission-plugins=ResourceQuota // and is not possible to start it from TEST_ARGS // Above tests are performed by doSheduletTests() and doPodResizeResourceQuotaTests() -// in test/node/pod_resize_test.go +// in test/e2e/node/pod_resize.go var _ = SIGDescribe("Pod InPlace Resize Container", framework.WithSerial(), feature.InPlacePodVerticalScaling, "[NodeAlphaFeature:InPlacePodVerticalScaling]", func() { if !podOnCgroupv2Node { From b8897e688d59c86caa4baeb553988183677fb265 Mon Sep 17 00:00:00 2001 From: Anish Shah Date: Thu, 17 Oct 2024 05:12:12 -0700 Subject: [PATCH 2/4] test: refactor duplicate IPPR e22 tests. This change refactors duplicate IPPR cluster and node e2e tests under test/e2e/common directory --- .../common/node/pod_resize.go} | 176 +-- test/e2e/node/pod_resize.go | 1025 +---------------- 2 files changed, 94 insertions(+), 1107 deletions(-) rename test/{e2e_node/pod_resize_test.go => e2e/common/node/pod_resize.go} (91%) diff --git a/test/e2e_node/pod_resize_test.go b/test/e2e/common/node/pod_resize.go similarity index 91% rename from test/e2e_node/pod_resize_test.go rename to test/e2e/common/node/pod_resize.go index 56f7f35315a..f5852e43889 100644 --- a/test/e2e_node/pod_resize_test.go +++ b/test/e2e/common/node/pod_resize.go @@ -14,10 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2enode +package node import ( "context" + "encoding/json" "fmt" "regexp" "strconv" @@ -32,14 +33,16 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" clientset "k8s.io/client-go/kubernetes" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -59,10 +62,7 @@ const ( ) var ( - podOnCgroupv2Node bool = IsCgroup2UnifiedMode() - cgroupMemLimit string = Cgroupv2MemLimit - cgroupCPULimit string = Cgroupv2CPULimit - cgroupCPURequest string = Cgroupv2CPURequest + podOnCgroupv2Node *bool ) type ContainerResources struct { @@ -114,16 +114,19 @@ type patchSpec struct { } `json:"spec"` } -func supportsInPlacePodVerticalScaling(ctx context.Context, f *framework.Framework) bool { - node := getLocalNode(ctx, f) +func isInPlacePodVerticalScalingSupportedByRuntime(ctx context.Context, c clientset.Interface) bool { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, c) + framework.ExpectNoError(err) re := regexp.MustCompile("containerd://(.*)") match := re.FindStringSubmatch(node.Status.NodeInfo.ContainerRuntimeVersion) if len(match) != 2 { return false } - // TODO(InPlacePodVerticalScaling): Update when RuntimeHandlerFeature for pod resize have been implemented if ver, verr := semver.ParseTolerant(match[1]); verr == nil { - return ver.Compare(semver.MustParse(MinContainerRuntimeVersion)) >= 0 + if ver.Compare(semver.MustParse(MinContainerRuntimeVersion)) < 0 { + return false + } + return true } return false } @@ -222,15 +225,11 @@ func makeTestContainer(tcInfo TestContainerInfo) (v1.Container, v1.ContainerStat func makeTestPod(ns, name, timeStamp string, tcInfo []TestContainerInfo) *v1.Pod { var testContainers []v1.Container - var podOS *v1.PodOS for _, ci := range tcInfo { tc, _ := makeTestContainer(ci) testContainers = append(testContainers, tc) } - - podOS = &v1.PodOS{Name: v1.Linux} - pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -240,7 +239,7 @@ func makeTestPod(ns, name, timeStamp string, tcInfo []TestContainerInfo) *v1.Pod }, }, Spec: v1.PodSpec{ - OS: podOS, + OS: &v1.PodOS{Name: v1.Linux}, Containers: testContainers, RestartPolicy: v1.RestartPolicyOnFailure, }, @@ -248,89 +247,95 @@ func makeTestPod(ns, name, timeStamp string, tcInfo []TestContainerInfo) *v1.Pod return pod } -func verifyPodResizePolicy(pod *v1.Pod, tcInfo []TestContainerInfo) { +func verifyPodResizePolicy(gotPod *v1.Pod, wantCtrs []TestContainerInfo) { ginkgo.GinkgoHelper() - cMap := make(map[string]*v1.Container) - for i, c := range pod.Spec.Containers { - cMap[c.Name] = &pod.Spec.Containers[i] - } - for _, ci := range tcInfo { - gomega.Expect(cMap).Should(gomega.HaveKey(ci.Name)) - c := cMap[ci.Name] - tc, _ := makeTestContainer(ci) - gomega.Expect(tc.ResizePolicy).To(gomega.Equal(c.ResizePolicy)) + for i, wantCtr := range wantCtrs { + gotCtr := &gotPod.Spec.Containers[i] + ctr, _ := makeTestContainer(wantCtr) + gomega.Expect(gotCtr.Name).To(gomega.Equal(ctr.Name)) + gomega.Expect(gotCtr.ResizePolicy).To(gomega.Equal(ctr.ResizePolicy)) } } -func verifyPodResources(pod *v1.Pod, tcInfo []TestContainerInfo) { +func verifyPodResources(gotPod *v1.Pod, wantCtrs []TestContainerInfo) { ginkgo.GinkgoHelper() - cMap := make(map[string]*v1.Container) - for i, c := range pod.Spec.Containers { - cMap[c.Name] = &pod.Spec.Containers[i] - } - for _, ci := range tcInfo { - gomega.Expect(cMap).Should(gomega.HaveKey(ci.Name)) - c := cMap[ci.Name] - tc, _ := makeTestContainer(ci) - gomega.Expect(tc.Resources).To(gomega.Equal(c.Resources)) + for i, wantCtr := range wantCtrs { + gotCtr := &gotPod.Spec.Containers[i] + ctr, _ := makeTestContainer(wantCtr) + gomega.Expect(gotCtr.Name).To(gomega.Equal(ctr.Name)) + gomega.Expect(gotCtr.Resources).To(gomega.Equal(ctr.Resources)) } } -func verifyPodAllocations(pod *v1.Pod, tcInfo []TestContainerInfo) error { +func verifyPodAllocations(gotPod *v1.Pod, wantCtrs []TestContainerInfo) error { ginkgo.GinkgoHelper() - cStatusMap := make(map[string]*v1.ContainerStatus) - for i, c := range pod.Status.ContainerStatuses { - cStatusMap[c.Name] = &pod.Status.ContainerStatuses[i] - } - - for _, ci := range tcInfo { - gomega.Expect(cStatusMap).Should(gomega.HaveKey(ci.Name)) - cStatus := cStatusMap[ci.Name] - if ci.Allocations == nil { - if ci.Resources != nil { - alloc := &ContainerAllocations{CPUAlloc: ci.Resources.CPUReq, MemAlloc: ci.Resources.MemReq} - ci.Allocations = alloc + for i, wantCtr := range wantCtrs { + gotCtrStatus := &gotPod.Status.ContainerStatuses[i] + if wantCtr.Allocations == nil { + if wantCtr.Resources != nil { + alloc := &ContainerAllocations{CPUAlloc: wantCtr.Resources.CPUReq, MemAlloc: wantCtr.Resources.MemReq} + wantCtr.Allocations = alloc defer func() { - ci.Allocations = nil + wantCtr.Allocations = nil }() } } - _, tcStatus := makeTestContainer(ci) - if !cmp.Equal(cStatus.AllocatedResources, tcStatus.AllocatedResources) { + _, ctrStatus := makeTestContainer(wantCtr) + gomega.Expect(gotCtrStatus.Name).To(gomega.Equal(ctrStatus.Name)) + if !cmp.Equal(gotCtrStatus.AllocatedResources, ctrStatus.AllocatedResources) { return fmt.Errorf("failed to verify Pod allocations, allocated resources not equal to expected") } } return nil } -func verifyPodStatusResources(pod *v1.Pod, tcInfo []TestContainerInfo) { +func verifyPodStatusResources(gotPod *v1.Pod, wantCtrs []TestContainerInfo) { ginkgo.GinkgoHelper() - csMap := make(map[string]*v1.ContainerStatus) - for i, c := range pod.Status.ContainerStatuses { - csMap[c.Name] = &pod.Status.ContainerStatuses[i] + for i, wantCtr := range wantCtrs { + gotCtrStatus := &gotPod.Status.ContainerStatuses[i] + ctr, _ := makeTestContainer(wantCtr) + gomega.Expect(gotCtrStatus.Name).To(gomega.Equal(ctr.Name)) + gomega.Expect(ctr.Resources).To(gomega.Equal(*gotCtrStatus.Resources)) } - for _, ci := range tcInfo { - gomega.Expect(csMap).Should(gomega.HaveKey(ci.Name)) - cs := csMap[ci.Name] - tc, _ := makeTestContainer(ci) - gomega.Expect(tc.Resources).To(gomega.Equal(*cs.Resources)) +} + +func isPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) bool { + // Determine if pod is running on cgroupv2 or cgroupv1 node + //TODO(vinaykul,InPlacePodVerticalScaling): Is there a better way to determine this? + cmd := "mount -t cgroup2" + out, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd) + if err != nil { + return false } + return len(out) != 0 } func verifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework, pod *v1.Pod, tcInfo []TestContainerInfo) error { ginkgo.GinkgoHelper() + if podOnCgroupv2Node == nil { + value := isPodOnCgroupv2Node(f, pod) + podOnCgroupv2Node = &value + } + cgroupMemLimit := Cgroupv2MemLimit + cgroupCPULimit := Cgroupv2CPULimit + cgroupCPURequest := Cgroupv2CPURequest + if !*podOnCgroupv2Node { + cgroupMemLimit = CgroupMemLimit + cgroupCPULimit = CgroupCPUQuota + cgroupCPURequest = CgroupCPUShares + } verifyCgroupValue := func(cName, cgPath, expectedCgValue string) error { - mycmd := fmt.Sprintf("head -n 1 %s", cgPath) - cgValue, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", mycmd) + cmd := fmt.Sprintf("head -n 1 %s", cgPath) framework.Logf("Namespace %s Pod %s Container %s - looking for cgroup value %s in path %s", pod.Namespace, pod.Name, cName, expectedCgValue, cgPath) + cgValue, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd) if err != nil { - return fmt.Errorf("failed to find expected value '%s' in container cgroup '%s'", expectedCgValue, cgPath) + return fmt.Errorf("failed to find expected value %q in container cgroup %q", expectedCgValue, cgPath) } cgValue = strings.Trim(cgValue, "\n") if cgValue != expectedCgValue { - return fmt.Errorf("cgroup value '%s' not equal to expected '%s'", cgValue, expectedCgValue) + return fmt.Errorf("cgroup value %q not equal to expected %q", cgValue, expectedCgValue) } return nil } @@ -356,7 +361,7 @@ func verifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework } expectedCPULimitString = strconv.FormatInt(cpuQuota, 10) expectedMemLimitString = strconv.FormatInt(expectedMemLimitInBytes, 10) - if podOnCgroupv2Node { + if *podOnCgroupv2Node { if expectedCPULimitString == "-1" { expectedCPULimitString = "max" } @@ -387,10 +392,17 @@ func verifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework return nil } -func waitForContainerRestart(ctx context.Context, f *framework.Framework, podClient *e2epod.PodClient, pod *v1.Pod, expectedContainers []TestContainerInfo) error { +func waitForContainerRestart(ctx context.Context, podClient *e2epod.PodClient, pod *v1.Pod, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, isRollback bool) error { ginkgo.GinkgoHelper() var restartContainersExpected []string - for _, ci := range expectedContainers { + + restartContainers := expectedContainers + // if we're rolling back, extract restart counts from test case "expected" containers + if isRollback { + restartContainers = initialContainers + } + + for _, ci := range restartContainers { if ci.RestartCount > 0 { restartContainersExpected = append(restartContainersExpected, ci.Name) } @@ -398,6 +410,7 @@ func waitForContainerRestart(ctx context.Context, f *framework.Framework, podCli if len(restartContainersExpected) == 0 { return nil } + pod, err := podClient.Get(ctx, pod.Name, metav1.GetOptions{}) if err != nil { return err @@ -420,14 +433,14 @@ func waitForContainerRestart(ctx context.Context, f *framework.Framework, podCli } } -func waitForPodResizeActuation(ctx context.Context, f *framework.Framework, c clientset.Interface, podClient *e2epod.PodClient, pod, patchedPod *v1.Pod, expectedContainers []TestContainerInfo) *v1.Pod { +func waitForPodResizeActuation(ctx context.Context, f *framework.Framework, podClient *e2epod.PodClient, pod, patchedPod *v1.Pod, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, isRollback bool) *v1.Pod { ginkgo.GinkgoHelper() var resizedPod *v1.Pod var pErr error timeouts := framework.NewTimeoutContext() // Wait for container restart gomega.Eventually(ctx, waitForContainerRestart, timeouts.PodStartShort, timeouts.Poll). - WithArguments(f, podClient, pod, expectedContainers). + WithArguments(podClient, pod, expectedContainers, initialContainers, isRollback). ShouldNot(gomega.HaveOccurred(), "failed waiting for expected container restart") // Verify Pod Containers Cgroup Values gomega.Eventually(ctx, verifyPodContainersCgroupValues, timeouts.PodStartShort, timeouts.Poll). @@ -1285,13 +1298,12 @@ func doPodResizeTests() { for idx := range tests { tc := tests[idx] ginkgo.It(tc.name, func(ctx context.Context) { - ginkgo.By("waiting for the node to be ready", func() { - if !supportsInPlacePodVerticalScaling(ctx, f) || framework.NodeOSDistroIs("windows") || isRunningOnArm64() { + ginkgo.By("check if in place pod vertical scaling is supported", func() { + if !isInPlacePodVerticalScalingSupportedByRuntime(ctx, f.ClientSet) || framework.NodeOSDistroIs("windows") { e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") } }) - var testPod *v1.Pod - var patchedPod *v1.Pod + var testPod, patchedPod *v1.Pod var pErr error tStamp := strconv.Itoa(time.Now().Nanosecond()) @@ -1322,9 +1334,8 @@ func doPodResizeTests() { ginkgo.By("verifying initial pod resize policy is as expected") verifyPodResizePolicy(newPod, tc.containers) - ginkgo.By("verifying initial pod status resources") + ginkgo.By("verifying initial pod status resources are as expected") verifyPodStatusResources(newPod, tc.containers) - ginkgo.By("verifying initial cgroup config are as expected") framework.ExpectNoError(verifyPodContainersCgroupValues(ctx, f, newPod, tc.containers)) @@ -1409,8 +1420,8 @@ func doPodResizeErrorTests() { for idx := range tests { tc := tests[idx] ginkgo.It(tc.name, func(ctx context.Context) { - ginkgo.By("waiting for the node to be ready", func() { - if !supportsInPlacePodVerticalScaling(ctx, f) || framework.NodeOSDistroIs("windows") || isRunningOnArm64() { + ginkgo.By("check if in place pod vertical scaling is supported", func() { + if !isInPlacePodVerticalScalingSupportedByRuntime(ctx, f.ClientSet) || framework.NodeOSDistroIs("windows") { e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") } }) @@ -1426,10 +1437,6 @@ func doPodResizeErrorTests() { ginkgo.By("creating pod") newPod := podClient.CreateSync(ctx, testPod) - perr := e2epod.WaitForPodCondition(ctx, f.ClientSet, newPod.Namespace, newPod.Name, "Ready", timeouts.PodStartSlow, testutils.PodRunningReady) - framework.ExpectNoError(perr, "pod %s/%s did not go running", newPod.Namespace, newPod.Name) - framework.Logf("pod %s/%s running", newPod.Namespace, newPod.Name) - ginkgo.By("verifying initial pod resources, allocations, and policy are as expected") verifyPodResources(newPod, tc.containers) verifyPodResizePolicy(newPod, tc.containers) @@ -1469,12 +1476,7 @@ func doPodResizeErrorTests() { // Above tests are performed by doSheduletTests() and doPodResizeResourceQuotaTests() // in test/e2e/node/pod_resize.go -var _ = SIGDescribe("Pod InPlace Resize Container", framework.WithSerial(), feature.InPlacePodVerticalScaling, "[NodeAlphaFeature:InPlacePodVerticalScaling]", func() { - if !podOnCgroupv2Node { - cgroupMemLimit = CgroupMemLimit - cgroupCPULimit = CgroupCPUQuota - cgroupCPURequest = CgroupCPUShares - } +var _ = SIGDescribe("Pod InPlace Resize Container", framework.WithSerial(), feature.InPlacePodVerticalScaling, func() { doPodResizeTests() doPodResizeErrorTests() }) diff --git a/test/e2e/node/pod_resize.go b/test/e2e/node/pod_resize.go index 464981e3019..9a087b5760b 100644 --- a/test/e2e/node/pod_resize.go +++ b/test/e2e/node/pod_resize.go @@ -26,17 +26,19 @@ import ( "strings" "time" + semver "github.com/blang/semver/v4" + "github.com/google/go-cmp/cmp" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" clientset "k8s.io/client-go/kubernetes" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" resourceapi "k8s.io/kubernetes/pkg/api/v1/resource" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" @@ -44,11 +46,6 @@ import ( e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" imageutils "k8s.io/kubernetes/test/utils/image" - - semver "github.com/blang/semver/v4" - "github.com/google/go-cmp/cmp" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" ) const ( @@ -583,910 +580,6 @@ func genPatchString(containers []TestContainerInfo) (string, error) { return string(patchBytes), nil } -func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error { - oldData, err := json.Marshal(old) - if err != nil { - return err - } - - newData, err := json.Marshal(new) - if err != nil { - return err - } - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{}) - if err != nil { - return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err) - } - _, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") - return err -} - -func addExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string, extendedResourceQuantity resource.Quantity) { - extendedResource := v1.ResourceName(extendedResourceName) - - ginkgo.By("Adding a custom resource") - OriginalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - framework.ExpectNoError(err) - - node := OriginalNode.DeepCopy() - node.Status.Capacity[extendedResource] = extendedResourceQuantity - node.Status.Allocatable[extendedResource] = extendedResourceQuantity - err = patchNode(context.Background(), clientSet, OriginalNode.DeepCopy(), node) - framework.ExpectNoError(err) - - gomega.Eventually(func() error { - node, err = clientSet.CoreV1().Nodes().Get(context.Background(), node.Name, metav1.GetOptions{}) - framework.ExpectNoError(err) - - fakeResourceCapacity, exists := node.Status.Capacity[extendedResource] - if !exists { - return fmt.Errorf("node %s has no %s resource capacity", node.Name, extendedResourceName) - } - if expectedResource := resource.MustParse("123"); fakeResourceCapacity.Cmp(expectedResource) != 0 { - return fmt.Errorf("node %s has resource capacity %s, expected: %s", node.Name, fakeResourceCapacity.String(), expectedResource.String()) - } - - return nil - }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) -} - -func removeExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string) { - extendedResource := v1.ResourceName(extendedResourceName) - - ginkgo.By("Removing a custom resource") - originalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - framework.ExpectNoError(err) - - node := originalNode.DeepCopy() - delete(node.Status.Capacity, extendedResource) - delete(node.Status.Allocatable, extendedResource) - err = patchNode(context.Background(), clientSet, originalNode.DeepCopy(), node) - framework.ExpectNoError(err) - - gomega.Eventually(func() error { - node, err = clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - framework.ExpectNoError(err) - - if _, exists := node.Status.Capacity[extendedResource]; exists { - return fmt.Errorf("node %s has resource capacity %s which is expected to be removed", node.Name, extendedResourceName) - } - - return nil - }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) -} - -func doPodResizeTests() { - f := framework.NewDefaultFramework("pod-resize") - var podClient *e2epod.PodClient - - ginkgo.BeforeEach(func() { - podClient = e2epod.NewPodClient(f) - }) - - type testCase struct { - name string - containers []TestContainerInfo - patchString string - expected []TestContainerInfo - addExtendedResource bool - } - - noRestart := v1.NotRequired - doRestart := v1.RestartContainer - tests := []testCase{ - { - name: "Guaranteed QoS pod, one container - increase CPU & memory", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - }, - { - name: "Guaranteed QoS pod, one container - decrease CPU & memory", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "500Mi", MemLim: "500Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"100m","memory":"250Mi"},"limits":{"cpu":"100m","memory":"250Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "250Mi", MemLim: "250Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - }, - { - name: "Guaranteed QoS pod, one container - increase CPU & decrease memory", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"100Mi"},"limits":{"cpu":"200m","memory":"100Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "100Mi", MemLim: "100Mi"}, - }, - }, - }, - { - name: "Guaranteed QoS pod, one container - decrease CPU & increase memory", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"300Mi"},"limits":{"cpu":"50m","memory":"300Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "50m", CPULim: "50m", MemReq: "300Mi", MemLim: "300Mi"}, - }, - }, - }, - { - name: "Guaranteed QoS pod, three containers (c1, c2, c3) - increase: CPU (c1,c3), memory (c2) ; decrease: CPU (c2), memory (c1,c3)", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - { - Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "200Mi", MemLim: "200Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - { - Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"140m","memory":"50Mi"},"limits":{"cpu":"140m","memory":"50Mi"}}}, - {"name":"c2", "resources":{"requests":{"cpu":"150m","memory":"240Mi"},"limits":{"cpu":"150m","memory":"240Mi"}}}, - {"name":"c3", "resources":{"requests":{"cpu":"340m","memory":"250Mi"},"limits":{"cpu":"340m","memory":"250Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "140m", CPULim: "140m", MemReq: "50Mi", MemLim: "50Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - { - Name: "c2", - Resources: &ContainerResources{CPUReq: "150m", CPULim: "150m", MemReq: "240Mi", MemLim: "240Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - { - Name: "c3", - Resources: &ContainerResources{CPUReq: "340m", CPULim: "340m", MemReq: "250Mi", MemLim: "250Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests only", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"200Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory limits only", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"limits":{"memory":"400Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "400Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests only", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"300Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "300Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory limits only", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"limits":{"memory":"600Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "600Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests only", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"100m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU limits only", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"limits":{"cpu":"300m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests only", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"150m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "150m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU limits only", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"limits":{"cpu":"500m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "500m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"cpu":"200m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"cpu":"400m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and increase CPU limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"cpu":"500m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "500m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and decrease CPU limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"cpu":"300m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "250Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"memory":"300Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "300Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"memory":"500Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "300Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and increase memory limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"memory":"500Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and decrease memory limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"memory":"300Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "300Mi", MemLim: "300Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and increase memory limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"memory":"500Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and decrease memory limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"memory":"400Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and increase CPU limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"cpu":"300m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "300m", MemReq: "100Mi", MemLim: "400Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and decrease CPU limits", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"cpu":"300m"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "300Mi", MemLim: "400Mi"}, - }, - }, - }, - { - name: "Burstable QoS pod, one container with cpu & memory requests - decrease memory request", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", MemReq: "500Mi"}, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"400Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", MemReq: "400Mi"}, - }, - }, - }, - { - name: "Guaranteed QoS pod, one container - increase CPU (NotRequired) & memory (RestartContainer)", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &doRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &doRestart, - RestartCount: 1, - }, - }, - }, - { - name: "Burstable QoS pod, one container - decrease CPU (RestartContainer) & memory (NotRequired)", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, - CPUPolicy: &doRestart, - MemPolicy: &noRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"100Mi"},"limits":{"cpu":"100m","memory":"200Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "100Mi", MemLim: "200Mi"}, - CPUPolicy: &doRestart, - MemPolicy: &noRestart, - RestartCount: 1, - }, - }, - }, - { - name: "Burstable QoS pod, three containers - increase c1 resources, no change for c2, decrease c3 resources (no net change for pod)", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - { - Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &doRestart, - }, - { - Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"150m","memory":"150Mi"},"limits":{"cpu":"250m","memory":"250Mi"}}}, - {"name":"c3", "resources":{"requests":{"cpu":"250m","memory":"250Mi"},"limits":{"cpu":"350m","memory":"350Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "150m", CPULim: "250m", MemReq: "150Mi", MemLim: "250Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - { - Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &doRestart, - }, - { - Name: "c3", - Resources: &ContainerResources{CPUReq: "250m", CPULim: "350m", MemReq: "250Mi", MemLim: "350Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - }, - { - name: "Burstable QoS pod, three containers - decrease c1 resources, increase c2 resources, no change for c3 (net increase for pod)", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - { - Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &doRestart, - }, - { - Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"50Mi"},"limits":{"cpu":"150m","memory":"150Mi"}}}, - {"name":"c2", "resources":{"requests":{"cpu":"350m","memory":"350Mi"},"limits":{"cpu":"450m","memory":"450Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "50m", CPULim: "150m", MemReq: "50Mi", MemLim: "150Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - { - Name: "c2", - Resources: &ContainerResources{CPUReq: "350m", CPULim: "450m", MemReq: "350Mi", MemLim: "450Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &doRestart, - RestartCount: 1, - }, - { - Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - }, - { - name: "Burstable QoS pod, three containers - no change for c1, increase c2 resources, decrease c3 (net decrease for pod)", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, - CPUPolicy: &doRestart, - MemPolicy: &doRestart, - }, - { - Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, - CPUPolicy: &doRestart, - MemPolicy: &noRestart, - }, - { - Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &doRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c2", "resources":{"requests":{"cpu":"250m","memory":"250Mi"},"limits":{"cpu":"350m","memory":"350Mi"}}}, - {"name":"c3", "resources":{"requests":{"cpu":"100m","memory":"100Mi"},"limits":{"cpu":"200m","memory":"200Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, - CPUPolicy: &doRestart, - MemPolicy: &doRestart, - }, - { - Name: "c2", - Resources: &ContainerResources{CPUReq: "250m", CPULim: "350m", MemReq: "250Mi", MemLim: "350Mi"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - RestartCount: 1, - }, - { - Name: "c3", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, - CPUPolicy: &doRestart, - MemPolicy: &doRestart, - RestartCount: 1, - }, - }, - }, - { - name: "Guaranteed QoS pod, one container - increase CPU & memory with an extended resource", - containers: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi", - ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} - ]}}`, - expected: []TestContainerInfo{ - { - Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi", - ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, - CPUPolicy: &noRestart, - MemPolicy: &noRestart, - }, - }, - addExtendedResource: true, - }, - } - - for idx := range tests { - tc := tests[idx] - ginkgo.It(tc.name, func(ctx context.Context) { - var testPod, patchedPod *v1.Pod - var pErr error - - tStamp := strconv.Itoa(time.Now().Nanosecond()) - initDefaultResizePolicy(tc.containers) - initDefaultResizePolicy(tc.expected) - testPod = makeTestPod(f.Namespace.Name, "testpod", tStamp, tc.containers) - - if tc.addExtendedResource { - nodes, err := e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) - framework.ExpectNoError(err) - - for _, node := range nodes.Items { - addExtendedResource(f.ClientSet, node.Name, fakeExtendedResource, resource.MustParse("123")) - } - defer func() { - for _, node := range nodes.Items { - removeExtendedResource(f.ClientSet, node.Name, fakeExtendedResource) - } - }() - } - - ginkgo.By("creating pod") - newPod := podClient.CreateSync(ctx, testPod) - - ginkgo.By("verifying the pod is in kubernetes") - selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": tStamp})) - options := metav1.ListOptions{LabelSelector: selector.String()} - podList, err := podClient.List(context.TODO(), options) - framework.ExpectNoError(err, "failed to query for pods") - gomega.Expect(podList.Items).Should(gomega.HaveLen(1)) - - ginkgo.By("verifying initial pod resources, allocations, and policy are as expected") - verifyPodResources(newPod, tc.containers) - verifyPodResizePolicy(newPod, tc.containers) - - ginkgo.By("verifying initial pod status resources and cgroup config are as expected") - verifyPodStatusResources(newPod, tc.containers) - // Check cgroup values only for containerd versions before 1.6.9 - if !isInPlaceResizeSupportedByRuntime(f.ClientSet, newPod.Spec.NodeName) { - if !framework.NodeOSDistroIs("windows") { - verifyPodContainersCgroupValues(newPod, tc.containers, true) - } - } - - patchAndVerify := func(patchString string, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, opStr string, isRollback bool) { - ginkgo.By(fmt.Sprintf("patching pod for %s", opStr)) - patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(context.TODO(), newPod.Name, - types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}) - framework.ExpectNoError(pErr, fmt.Sprintf("failed to patch pod for %s", opStr)) - - ginkgo.By(fmt.Sprintf("verifying pod patched for %s", opStr)) - verifyPodResources(patchedPod, expectedContainers) - verifyPodAllocations(patchedPod, initialContainers, true) - - ginkgo.By(fmt.Sprintf("waiting for %s to be actuated", opStr)) - resizedPod := waitForPodResizeActuation(f.ClientSet, podClient, newPod, patchedPod, expectedContainers, initialContainers, isRollback) - - // Check cgroup values only for containerd versions before 1.6.9 - if !isInPlaceResizeSupportedByRuntime(f.ClientSet, newPod.Spec.NodeName) { - ginkgo.By(fmt.Sprintf("verifying pod container's cgroup values after %s", opStr)) - if !framework.NodeOSDistroIs("windows") { - verifyPodContainersCgroupValues(resizedPod, expectedContainers, true) - } - } - - ginkgo.By(fmt.Sprintf("verifying pod resources after %s", opStr)) - verifyPodResources(resizedPod, expectedContainers) - - ginkgo.By(fmt.Sprintf("verifying pod allocations after %s", opStr)) - verifyPodAllocations(resizedPod, expectedContainers, true) - } - - patchAndVerify(tc.patchString, tc.expected, tc.containers, "resize", false) - - rbPatchStr, err := genPatchString(tc.containers) - framework.ExpectNoError(err) - // Resize has been actuated, test rollback - patchAndVerify(rbPatchStr, tc.containers, tc.expected, "rollback", true) - - ginkgo.By("deleting pod") - err = e2epod.DeletePodWithWait(ctx, f.ClientSet, newPod) - framework.ExpectNoError(err, "failed to delete pod") - }) - } -} - func doPodResizeResourceQuotaTests() { f := framework.NewDefaultFramework("pod-resize-resource-quota") var podClient *e2epod.PodClient @@ -1543,13 +636,6 @@ func doPodResizeResourceQuotaTests() { newPod1 := podClient.CreateSync(ctx, testPod1) newPod2 := podClient.CreateSync(ctx, testPod2) - ginkgo.By("verifying the pod is in kubernetes") - selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": tStamp})) - options := metav1.ListOptions{LabelSelector: selector.String()} - podList, listErr := podClient.List(context.TODO(), options) - framework.ExpectNoError(listErr, "failed to query for pods") - gomega.Expect(podList.Items).Should(gomega.HaveLen(2)) - ginkgo.By("verifying initial pod resources, allocations, and policy are as expected") verifyPodResources(newPod1, containers) @@ -1609,104 +695,6 @@ func doPodResizeResourceQuotaTests() { }) } -func doPodResizeErrorTests() { - f := framework.NewDefaultFramework("pod-resize-errors") - var podClient *e2epod.PodClient - ginkgo.BeforeEach(func() { - podClient = e2epod.NewPodClient(f) - }) - - type testCase struct { - name string - containers []TestContainerInfo - patchString string - patchError string - expected []TestContainerInfo - } - - tests := []testCase{ - { - name: "BestEffort pod - try requesting memory, expect error", - containers: []TestContainerInfo{ - { - Name: "c1", - }, - }, - patchString: `{"spec":{"containers":[ - {"name":"c1", "resources":{"requests":{"memory":"400Mi"}}} - ]}}`, - patchError: "Pod QoS is immutable", - expected: []TestContainerInfo{ - { - Name: "c1", - }, - }, - }, - } - - for idx := range tests { - tc := tests[idx] - ginkgo.It(tc.name, func(ctx context.Context) { - var testPod, patchedPod *v1.Pod - var pErr error - - tStamp := strconv.Itoa(time.Now().Nanosecond()) - initDefaultResizePolicy(tc.containers) - initDefaultResizePolicy(tc.expected) - testPod = makeTestPod(f.Namespace.Name, "testpod", tStamp, tc.containers) - - ginkgo.By("creating pod") - newPod := podClient.CreateSync(ctx, testPod) - - ginkgo.By("verifying the pod is in kubernetes") - selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": tStamp})) - options := metav1.ListOptions{LabelSelector: selector.String()} - podList, err := podClient.List(context.TODO(), options) - framework.ExpectNoError(err, "failed to query for pods") - gomega.Expect(podList.Items).Should(gomega.HaveLen(1)) - - ginkgo.By("verifying initial pod resources, allocations, and policy are as expected") - verifyPodResources(newPod, tc.containers) - verifyPodResizePolicy(newPod, tc.containers) - - ginkgo.By("verifying initial pod status resources and cgroup config are as expected") - verifyPodStatusResources(newPod, tc.containers) - if !isInPlaceResizeSupportedByRuntime(f.ClientSet, newPod.Spec.NodeName) { - if !framework.NodeOSDistroIs("windows") { - verifyPodContainersCgroupValues(newPod, tc.containers, true) - } - } - - ginkgo.By("patching pod for resize") - patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(context.TODO(), newPod.Name, - types.StrategicMergePatchType, []byte(tc.patchString), metav1.PatchOptions{}) - if tc.patchError == "" { - framework.ExpectNoError(pErr, "failed to patch pod for resize") - } else { - gomega.Expect(pErr).To(gomega.HaveOccurred(), tc.patchError) - patchedPod = newPod - } - - if !isInPlaceResizeSupportedByRuntime(f.ClientSet, patchedPod.Spec.NodeName) { - ginkgo.By("verifying pod container's cgroup values after patch") - if !framework.NodeOSDistroIs("windows") { - verifyPodContainersCgroupValues(patchedPod, tc.expected, true) - } - } - - ginkgo.By("verifying pod resources after patch") - verifyPodResources(patchedPod, tc.expected) - - ginkgo.By("verifying pod allocations after patch") - verifyPodAllocations(patchedPod, tc.expected, true) - - ginkgo.By("deleting pod") - err = e2epod.DeletePodWithWait(ctx, f.ClientSet, newPod) - framework.ExpectNoError(err, "failed to delete pod") - }) - } -} - func doPodResizeSchedulerTests() { f := framework.NewDefaultFramework("pod-resize-scheduler") var podClient *e2epod.PodClient @@ -1811,7 +799,6 @@ func doPodResizeSchedulerTests() { ginkgo.By(fmt.Sprintf("TEST1: Verify that pod '%s' is running after resize", testPod2.Name)) framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, testPod2)) - // // Scheduler focussed pod resize E2E test case #2 // 1. With pod1 + pod2 running on node above, create pod3 that requests more CPU than available, verify pending. // 2. Resize pod1 down so that pod3 gets room to be scheduled. @@ -1878,7 +865,5 @@ var _ = SIGDescribe(framework.WithSerial(), "Pod InPlace Resize Container (sched }) var _ = SIGDescribe("Pod InPlace Resize Container", feature.InPlacePodVerticalScaling, func() { - doPodResizeTests() doPodResizeResourceQuotaTests() - doPodResizeErrorTests() }) From d72c7319f8ea7e4081e54c736a0fc88edbddb523 Mon Sep 17 00:00:00 2001 From: Anish Shah Date: Thu, 17 Oct 2024 03:29:51 -0700 Subject: [PATCH 3/4] test: refactor duplicate inplace pod resize test utilities --- test/e2e/common/node/pod_resize.go | 791 +++++++---------------------- test/e2e/framework/node/helper.go | 13 +- test/e2e/framework/pod/resize.go | 470 +++++++++++++++++ test/e2e/node/pod_resize.go | 657 +++--------------------- 4 files changed, 729 insertions(+), 1202 deletions(-) create mode 100644 test/e2e/framework/pod/resize.go diff --git a/test/e2e/common/node/pod_resize.go b/test/e2e/common/node/pod_resize.go index f5852e43889..b067adf7c11 100644 --- a/test/e2e/common/node/pod_resize.go +++ b/test/e2e/common/node/pod_resize.go @@ -20,13 +20,9 @@ import ( "context" "encoding/json" "fmt" - "regexp" "strconv" - "strings" "time" - semver "github.com/blang/semver/v4" - "github.com/google/go-cmp/cmp" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" @@ -36,450 +32,17 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" clientset "k8s.io/client-go/kubernetes" - podutil "k8s.io/kubernetes/pkg/api/v1/pod" - kubecm "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - imageutils "k8s.io/kubernetes/test/utils/image" ) const ( - CgroupCPUPeriod string = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" - CgroupCPUShares string = "/sys/fs/cgroup/cpu/cpu.shares" - CgroupCPUQuota string = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" - CgroupMemLimit string = "/sys/fs/cgroup/memory/memory.limit_in_bytes" - Cgroupv2MemLimit string = "/sys/fs/cgroup/memory.max" - Cgroupv2MemRequest string = "/sys/fs/cgroup/memory.min" - Cgroupv2CPULimit string = "/sys/fs/cgroup/cpu.max" - Cgroupv2CPURequest string = "/sys/fs/cgroup/cpu.weight" - CPUPeriod string = "100000" - MinContainerRuntimeVersion string = "1.6.9" - fakeExtendedResource = "dummy.com/dummy" ) -var ( - podOnCgroupv2Node *bool -) - -type ContainerResources struct { - CPUReq string - CPULim string - MemReq string - MemLim string - EphStorReq string - EphStorLim string - ExtendedResourceReq string - ExtendedResourceLim string -} - -type ContainerAllocations struct { - CPUAlloc string - MemAlloc string - ephStorAlloc string - ExtendedResourceAlloc string -} - -type TestContainerInfo struct { - Name string - Resources *ContainerResources - Allocations *ContainerAllocations - CPUPolicy *v1.ResourceResizeRestartPolicy - MemPolicy *v1.ResourceResizeRestartPolicy - RestartCount int32 -} - -type containerPatch struct { - Name string `json:"name"` - Resources struct { - Requests struct { - CPU string `json:"cpu,omitempty"` - Memory string `json:"memory,omitempty"` - EphStor string `json:"ephemeral-storage,omitempty"` - } `json:"requests"` - Limits struct { - CPU string `json:"cpu,omitempty"` - Memory string `json:"memory,omitempty"` - EphStor string `json:"ephemeral-storage,omitempty"` - } `json:"limits"` - } `json:"resources"` -} - -type patchSpec struct { - Spec struct { - Containers []containerPatch `json:"containers"` - } `json:"spec"` -} - -func isInPlacePodVerticalScalingSupportedByRuntime(ctx context.Context, c clientset.Interface) bool { - node, err := e2enode.GetRandomReadySchedulableNode(ctx, c) - framework.ExpectNoError(err) - re := regexp.MustCompile("containerd://(.*)") - match := re.FindStringSubmatch(node.Status.NodeInfo.ContainerRuntimeVersion) - if len(match) != 2 { - return false - } - if ver, verr := semver.ParseTolerant(match[1]); verr == nil { - if ver.Compare(semver.MustParse(MinContainerRuntimeVersion)) < 0 { - return false - } - return true - } - return false -} - -func getTestResourceInfo(tcInfo TestContainerInfo) (v1.ResourceRequirements, v1.ResourceList, []v1.ContainerResizePolicy) { - var res v1.ResourceRequirements - var alloc v1.ResourceList - var resizePol []v1.ContainerResizePolicy - - if tcInfo.Resources != nil { - var lim, req v1.ResourceList - if tcInfo.Resources.CPULim != "" || tcInfo.Resources.MemLim != "" || tcInfo.Resources.EphStorLim != "" { - lim = make(v1.ResourceList) - } - if tcInfo.Resources.CPUReq != "" || tcInfo.Resources.MemReq != "" || tcInfo.Resources.EphStorReq != "" { - req = make(v1.ResourceList) - } - if tcInfo.Resources.CPULim != "" { - lim[v1.ResourceCPU] = resource.MustParse(tcInfo.Resources.CPULim) - } - if tcInfo.Resources.MemLim != "" { - lim[v1.ResourceMemory] = resource.MustParse(tcInfo.Resources.MemLim) - } - if tcInfo.Resources.EphStorLim != "" { - lim[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Resources.EphStorLim) - } - if tcInfo.Resources.CPUReq != "" { - req[v1.ResourceCPU] = resource.MustParse(tcInfo.Resources.CPUReq) - } - if tcInfo.Resources.MemReq != "" { - req[v1.ResourceMemory] = resource.MustParse(tcInfo.Resources.MemReq) - } - if tcInfo.Resources.EphStorReq != "" { - req[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Resources.EphStorReq) - } - res = v1.ResourceRequirements{Limits: lim, Requests: req} - } - if tcInfo.Allocations != nil { - alloc = make(v1.ResourceList) - if tcInfo.Allocations.CPUAlloc != "" { - alloc[v1.ResourceCPU] = resource.MustParse(tcInfo.Allocations.CPUAlloc) - } - if tcInfo.Allocations.MemAlloc != "" { - alloc[v1.ResourceMemory] = resource.MustParse(tcInfo.Allocations.MemAlloc) - } - if tcInfo.Allocations.ephStorAlloc != "" { - alloc[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Allocations.ephStorAlloc) - } - - } - if tcInfo.CPUPolicy != nil { - cpuPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceCPU, RestartPolicy: *tcInfo.CPUPolicy} - resizePol = append(resizePol, cpuPol) - } - if tcInfo.MemPolicy != nil { - memPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceMemory, RestartPolicy: *tcInfo.MemPolicy} - resizePol = append(resizePol, memPol) - } - return res, alloc, resizePol -} - -func initDefaultResizePolicy(containers []TestContainerInfo) { - noRestart := v1.NotRequired - setDefaultPolicy := func(ci *TestContainerInfo) { - if ci.CPUPolicy == nil { - ci.CPUPolicy = &noRestart - } - if ci.MemPolicy == nil { - ci.MemPolicy = &noRestart - } - } - for i := range containers { - setDefaultPolicy(&containers[i]) - } -} - -func makeTestContainer(tcInfo TestContainerInfo) (v1.Container, v1.ContainerStatus) { - cmd := "grep Cpus_allowed_list /proc/self/status | cut -f2 && sleep 1d" - res, alloc, resizePol := getTestResourceInfo(tcInfo) - - tc := v1.Container{ - Name: tcInfo.Name, - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"/bin/sh"}, - Args: []string{"-c", cmd}, - Resources: res, - ResizePolicy: resizePol, - } - - tcStatus := v1.ContainerStatus{ - Name: tcInfo.Name, - AllocatedResources: alloc, - } - return tc, tcStatus -} - -func makeTestPod(ns, name, timeStamp string, tcInfo []TestContainerInfo) *v1.Pod { - var testContainers []v1.Container - - for _, ci := range tcInfo { - tc, _ := makeTestContainer(ci) - testContainers = append(testContainers, tc) - } - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - Labels: map[string]string{ - "time": timeStamp, - }, - }, - Spec: v1.PodSpec{ - OS: &v1.PodOS{Name: v1.Linux}, - Containers: testContainers, - RestartPolicy: v1.RestartPolicyOnFailure, - }, - } - return pod -} - -func verifyPodResizePolicy(gotPod *v1.Pod, wantCtrs []TestContainerInfo) { - ginkgo.GinkgoHelper() - for i, wantCtr := range wantCtrs { - gotCtr := &gotPod.Spec.Containers[i] - ctr, _ := makeTestContainer(wantCtr) - gomega.Expect(gotCtr.Name).To(gomega.Equal(ctr.Name)) - gomega.Expect(gotCtr.ResizePolicy).To(gomega.Equal(ctr.ResizePolicy)) - } -} - -func verifyPodResources(gotPod *v1.Pod, wantCtrs []TestContainerInfo) { - ginkgo.GinkgoHelper() - for i, wantCtr := range wantCtrs { - gotCtr := &gotPod.Spec.Containers[i] - ctr, _ := makeTestContainer(wantCtr) - gomega.Expect(gotCtr.Name).To(gomega.Equal(ctr.Name)) - gomega.Expect(gotCtr.Resources).To(gomega.Equal(ctr.Resources)) - } -} - -func verifyPodAllocations(gotPod *v1.Pod, wantCtrs []TestContainerInfo) error { - ginkgo.GinkgoHelper() - for i, wantCtr := range wantCtrs { - gotCtrStatus := &gotPod.Status.ContainerStatuses[i] - if wantCtr.Allocations == nil { - if wantCtr.Resources != nil { - alloc := &ContainerAllocations{CPUAlloc: wantCtr.Resources.CPUReq, MemAlloc: wantCtr.Resources.MemReq} - wantCtr.Allocations = alloc - defer func() { - wantCtr.Allocations = nil - }() - } - } - - _, ctrStatus := makeTestContainer(wantCtr) - gomega.Expect(gotCtrStatus.Name).To(gomega.Equal(ctrStatus.Name)) - if !cmp.Equal(gotCtrStatus.AllocatedResources, ctrStatus.AllocatedResources) { - return fmt.Errorf("failed to verify Pod allocations, allocated resources not equal to expected") - } - } - return nil -} - -func verifyPodStatusResources(gotPod *v1.Pod, wantCtrs []TestContainerInfo) { - ginkgo.GinkgoHelper() - for i, wantCtr := range wantCtrs { - gotCtrStatus := &gotPod.Status.ContainerStatuses[i] - ctr, _ := makeTestContainer(wantCtr) - gomega.Expect(gotCtrStatus.Name).To(gomega.Equal(ctr.Name)) - gomega.Expect(ctr.Resources).To(gomega.Equal(*gotCtrStatus.Resources)) - } -} - -func isPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) bool { - // Determine if pod is running on cgroupv2 or cgroupv1 node - //TODO(vinaykul,InPlacePodVerticalScaling): Is there a better way to determine this? - cmd := "mount -t cgroup2" - out, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd) - if err != nil { - return false - } - return len(out) != 0 -} - -func verifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework, pod *v1.Pod, tcInfo []TestContainerInfo) error { - ginkgo.GinkgoHelper() - if podOnCgroupv2Node == nil { - value := isPodOnCgroupv2Node(f, pod) - podOnCgroupv2Node = &value - } - cgroupMemLimit := Cgroupv2MemLimit - cgroupCPULimit := Cgroupv2CPULimit - cgroupCPURequest := Cgroupv2CPURequest - if !*podOnCgroupv2Node { - cgroupMemLimit = CgroupMemLimit - cgroupCPULimit = CgroupCPUQuota - cgroupCPURequest = CgroupCPUShares - } - verifyCgroupValue := func(cName, cgPath, expectedCgValue string) error { - cmd := fmt.Sprintf("head -n 1 %s", cgPath) - framework.Logf("Namespace %s Pod %s Container %s - looking for cgroup value %s in path %s", - pod.Namespace, pod.Name, cName, expectedCgValue, cgPath) - cgValue, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd) - if err != nil { - return fmt.Errorf("failed to find expected value %q in container cgroup %q", expectedCgValue, cgPath) - } - cgValue = strings.Trim(cgValue, "\n") - if cgValue != expectedCgValue { - return fmt.Errorf("cgroup value %q not equal to expected %q", cgValue, expectedCgValue) - } - return nil - } - for _, ci := range tcInfo { - if ci.Resources == nil { - continue - } - tc, _ := makeTestContainer(ci) - if tc.Resources.Limits != nil || tc.Resources.Requests != nil { - var expectedCPUShares int64 - var expectedCPULimitString, expectedMemLimitString string - expectedMemLimitInBytes := tc.Resources.Limits.Memory().Value() - cpuRequest := tc.Resources.Requests.Cpu() - cpuLimit := tc.Resources.Limits.Cpu() - if cpuRequest.IsZero() && !cpuLimit.IsZero() { - expectedCPUShares = int64(kubecm.MilliCPUToShares(cpuLimit.MilliValue())) - } else { - expectedCPUShares = int64(kubecm.MilliCPUToShares(cpuRequest.MilliValue())) - } - cpuQuota := kubecm.MilliCPUToQuota(cpuLimit.MilliValue(), kubecm.QuotaPeriod) - if cpuLimit.IsZero() { - cpuQuota = -1 - } - expectedCPULimitString = strconv.FormatInt(cpuQuota, 10) - expectedMemLimitString = strconv.FormatInt(expectedMemLimitInBytes, 10) - if *podOnCgroupv2Node { - if expectedCPULimitString == "-1" { - expectedCPULimitString = "max" - } - expectedCPULimitString = fmt.Sprintf("%s %s", expectedCPULimitString, CPUPeriod) - if expectedMemLimitString == "0" { - expectedMemLimitString = "max" - } - // convert cgroup v1 cpu.shares value to cgroup v2 cpu.weight value - // https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2 - expectedCPUShares = int64(1 + ((expectedCPUShares-2)*9999)/262142) - } - if expectedMemLimitString != "0" { - err := verifyCgroupValue(ci.Name, cgroupMemLimit, expectedMemLimitString) - if err != nil { - return err - } - } - err := verifyCgroupValue(ci.Name, cgroupCPULimit, expectedCPULimitString) - if err != nil { - return err - } - err = verifyCgroupValue(ci.Name, cgroupCPURequest, strconv.FormatInt(expectedCPUShares, 10)) - if err != nil { - return err - } - } - } - return nil -} - -func waitForContainerRestart(ctx context.Context, podClient *e2epod.PodClient, pod *v1.Pod, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, isRollback bool) error { - ginkgo.GinkgoHelper() - var restartContainersExpected []string - - restartContainers := expectedContainers - // if we're rolling back, extract restart counts from test case "expected" containers - if isRollback { - restartContainers = initialContainers - } - - for _, ci := range restartContainers { - if ci.RestartCount > 0 { - restartContainersExpected = append(restartContainersExpected, ci.Name) - } - } - if len(restartContainersExpected) == 0 { - return nil - } - - pod, err := podClient.Get(ctx, pod.Name, metav1.GetOptions{}) - if err != nil { - return err - } - restartedContainersCount := 0 - for _, cName := range restartContainersExpected { - cs, _ := podutil.GetContainerStatus(pod.Status.ContainerStatuses, cName) - if cs.RestartCount < 1 { - break - } - restartedContainersCount++ - } - if restartedContainersCount == len(restartContainersExpected) { - return nil - } - if restartedContainersCount > len(restartContainersExpected) { - return fmt.Errorf("more container restarts than expected") - } else { - return fmt.Errorf("less container restarts than expected") - } -} - -func waitForPodResizeActuation(ctx context.Context, f *framework.Framework, podClient *e2epod.PodClient, pod, patchedPod *v1.Pod, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, isRollback bool) *v1.Pod { - ginkgo.GinkgoHelper() - var resizedPod *v1.Pod - var pErr error - timeouts := framework.NewTimeoutContext() - // Wait for container restart - gomega.Eventually(ctx, waitForContainerRestart, timeouts.PodStartShort, timeouts.Poll). - WithArguments(podClient, pod, expectedContainers, initialContainers, isRollback). - ShouldNot(gomega.HaveOccurred(), "failed waiting for expected container restart") - // Verify Pod Containers Cgroup Values - gomega.Eventually(ctx, verifyPodContainersCgroupValues, timeouts.PodStartShort, timeouts.Poll). - WithArguments(f, patchedPod, expectedContainers). - ShouldNot(gomega.HaveOccurred(), "failed to verify container cgroup values to match expected") - // Wait for pod resource allocations to equal expected values after resize - gomega.Eventually(ctx, func() error { - resizedPod, pErr = podClient.Get(ctx, pod.Name, metav1.GetOptions{}) - if pErr != nil { - return pErr - } - return verifyPodAllocations(resizedPod, expectedContainers) - }, timeouts.PodStartShort, timeouts.Poll). - ShouldNot(gomega.HaveOccurred(), "timed out waiting for pod resource allocation values to match expected") - return resizedPod -} - -func genPatchString(containers []TestContainerInfo) (string, error) { - var patch patchSpec - - for _, container := range containers { - var cPatch containerPatch - cPatch.Name = container.Name - cPatch.Resources.Requests.CPU = container.Resources.CPUReq - cPatch.Resources.Requests.Memory = container.Resources.MemReq - cPatch.Resources.Limits.CPU = container.Resources.CPULim - cPatch.Resources.Limits.Memory = container.Resources.MemLim - - patch.Spec.Containers = append(patch.Spec.Containers, cPatch) - } - - patchBytes, err := json.Marshal(patch) - if err != nil { - return "", err - } - - return string(patchBytes), nil -} - func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error { oldData, err := json.Marshal(old) if err != nil { @@ -561,9 +124,9 @@ func doPodResizeTests() { type testCase struct { name string - containers []TestContainerInfo + containers []e2epod.ResizableContainerInfo patchString string - expected []TestContainerInfo + expected []e2epod.ResizableContainerInfo addExtendedResource bool } @@ -572,10 +135,10 @@ func doPodResizeTests() { tests := []testCase{ { name: "Guaranteed QoS pod, one container - increase CPU & memory", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -583,10 +146,10 @@ func doPodResizeTests() { patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -594,10 +157,10 @@ func doPodResizeTests() { }, { name: "Guaranteed QoS pod, one container - decrease CPU & memory", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "500Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "500Mi", MemLim: "500Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -605,10 +168,10 @@ func doPodResizeTests() { patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"100m","memory":"250Mi"},"limits":{"cpu":"100m","memory":"250Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "250Mi", MemLim: "250Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "250Mi", MemLim: "250Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -616,58 +179,58 @@ func doPodResizeTests() { }, { name: "Guaranteed QoS pod, one container - increase CPU & decrease memory", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"100Mi"},"limits":{"cpu":"200m","memory":"100Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "100Mi", MemLim: "100Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "100Mi", MemLim: "100Mi"}, }, }, }, { name: "Guaranteed QoS pod, one container - decrease CPU & increase memory", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"300Mi"},"limits":{"cpu":"50m","memory":"300Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "50m", CPULim: "50m", MemReq: "300Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "50m", CPULim: "50m", MemReq: "300Mi", MemLim: "300Mi"}, }, }, }, { name: "Guaranteed QoS pod, three containers (c1, c2, c3) - increase: CPU (c1,c3), memory (c2) ; decrease: CPU (c2), memory (c1,c3)", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "100Mi", MemLim: "100Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, { Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "200Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "200Mi", MemLim: "200Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, { Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -677,22 +240,22 @@ func doPodResizeTests() { {"name":"c2", "resources":{"requests":{"cpu":"150m","memory":"240Mi"},"limits":{"cpu":"150m","memory":"240Mi"}}}, {"name":"c3", "resources":{"requests":{"cpu":"340m","memory":"250Mi"},"limits":{"cpu":"340m","memory":"250Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "140m", CPULim: "140m", MemReq: "50Mi", MemLim: "50Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "140m", CPULim: "140m", MemReq: "50Mi", MemLim: "50Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, { Name: "c2", - Resources: &ContainerResources{CPUReq: "150m", CPULim: "150m", MemReq: "240Mi", MemLim: "240Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "150m", CPULim: "150m", MemReq: "240Mi", MemLim: "240Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, { Name: "c3", - Resources: &ContainerResources{CPUReq: "340m", CPULim: "340m", MemReq: "250Mi", MemLim: "250Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "340m", CPULim: "340m", MemReq: "250Mi", MemLim: "250Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -700,388 +263,388 @@ func doPodResizeTests() { }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests only", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"200Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory limits only", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"limits":{"memory":"400Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "400Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests only", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"300Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "300Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "300Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory limits only", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"limits":{"memory":"600Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "600Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "600Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests only", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"100m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU limits only", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"limits":{"cpu":"300m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests only", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"150m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "150m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "150m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU limits only", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"limits":{"cpu":"500m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "500m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "500m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"cpu":"200m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"cpu":"400m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and increase CPU limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"cpu":"500m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "500m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "500m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and decrease CPU limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"cpu":"300m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "250Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "250Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"memory":"300Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "300Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"memory":"500Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "300Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "300Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and increase memory limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"memory":"500Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and decrease memory limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"memory":"300Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "300Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "300Mi", MemLim: "300Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease CPU requests and increase memory limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"100m"},"limits":{"memory":"500Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase CPU requests and decrease memory limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "400m", MemReq: "200Mi", MemLim: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"200m"},"limits":{"memory":"400Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - decrease memory requests and increase CPU limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"100Mi"},"limits":{"cpu":"300m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "300m", MemReq: "100Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "300m", MemReq: "100Mi", MemLim: "400Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests + limits - increase memory requests and decrease CPU limits", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "400m", MemReq: "200Mi", MemLim: "400Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"300Mi"},"limits":{"cpu":"300m"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "300Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "300Mi", MemLim: "400Mi"}, }, }, }, { name: "Burstable QoS pod, one container with cpu & memory requests - decrease memory request", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", MemReq: "500Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", MemReq: "500Mi"}, }, }, patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"memory":"400Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", MemReq: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", MemReq: "400Mi"}, }, }, }, { name: "Guaranteed QoS pod, one container - increase CPU (NotRequired) & memory (RestartContainer)", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi"}, CPUPolicy: &noRestart, MemPolicy: &doRestart, }, @@ -1089,10 +652,10 @@ func doPodResizeTests() { patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi"}, CPUPolicy: &noRestart, MemPolicy: &doRestart, RestartCount: 1, @@ -1101,10 +664,10 @@ func doPodResizeTests() { }, { name: "Burstable QoS pod, one container - decrease CPU (RestartContainer) & memory (NotRequired)", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "200Mi", MemLim: "400Mi"}, CPUPolicy: &doRestart, MemPolicy: &noRestart, }, @@ -1112,10 +675,10 @@ func doPodResizeTests() { patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"100Mi"},"limits":{"cpu":"100m","memory":"200Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "100Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "50m", CPULim: "100m", MemReq: "100Mi", MemLim: "200Mi"}, CPUPolicy: &doRestart, MemPolicy: &noRestart, RestartCount: 1, @@ -1124,22 +687,22 @@ func doPodResizeTests() { }, { name: "Burstable QoS pod, three containers - increase c1 resources, no change for c2, decrease c3 resources (no net change for pod)", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, { Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, CPUPolicy: &noRestart, MemPolicy: &doRestart, }, { Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -1148,22 +711,22 @@ func doPodResizeTests() { {"name":"c1", "resources":{"requests":{"cpu":"150m","memory":"150Mi"},"limits":{"cpu":"250m","memory":"250Mi"}}}, {"name":"c3", "resources":{"requests":{"cpu":"250m","memory":"250Mi"},"limits":{"cpu":"350m","memory":"350Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "150m", CPULim: "250m", MemReq: "150Mi", MemLim: "250Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "150m", CPULim: "250m", MemReq: "150Mi", MemLim: "250Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, { Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, CPUPolicy: &noRestart, MemPolicy: &doRestart, }, { Name: "c3", - Resources: &ContainerResources{CPUReq: "250m", CPULim: "350m", MemReq: "250Mi", MemLim: "350Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "250m", CPULim: "350m", MemReq: "250Mi", MemLim: "350Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -1171,22 +734,22 @@ func doPodResizeTests() { }, { name: "Burstable QoS pod, three containers - decrease c1 resources, increase c2 resources, no change for c3 (net increase for pod)", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, { Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, CPUPolicy: &noRestart, MemPolicy: &doRestart, }, { Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -1195,23 +758,23 @@ func doPodResizeTests() { {"name":"c1", "resources":{"requests":{"cpu":"50m","memory":"50Mi"},"limits":{"cpu":"150m","memory":"150Mi"}}}, {"name":"c2", "resources":{"requests":{"cpu":"350m","memory":"350Mi"},"limits":{"cpu":"450m","memory":"450Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "50m", CPULim: "150m", MemReq: "50Mi", MemLim: "150Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "50m", CPULim: "150m", MemReq: "50Mi", MemLim: "150Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, { Name: "c2", - Resources: &ContainerResources{CPUReq: "350m", CPULim: "450m", MemReq: "350Mi", MemLim: "450Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "350m", CPULim: "450m", MemReq: "350Mi", MemLim: "450Mi"}, CPUPolicy: &noRestart, MemPolicy: &doRestart, RestartCount: 1, }, { Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, }, @@ -1219,22 +782,22 @@ func doPodResizeTests() { }, { name: "Burstable QoS pod, three containers - no change for c1, increase c2 resources, decrease c3 (net decrease for pod)", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, CPUPolicy: &doRestart, MemPolicy: &doRestart, }, { Name: "c2", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "300m", MemReq: "200Mi", MemLim: "300Mi"}, CPUPolicy: &doRestart, MemPolicy: &noRestart, }, { Name: "c3", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "400m", MemReq: "300Mi", MemLim: "400Mi"}, CPUPolicy: &noRestart, MemPolicy: &doRestart, }, @@ -1243,23 +806,23 @@ func doPodResizeTests() { {"name":"c2", "resources":{"requests":{"cpu":"250m","memory":"250Mi"},"limits":{"cpu":"350m","memory":"350Mi"}}}, {"name":"c3", "resources":{"requests":{"cpu":"100m","memory":"100Mi"},"limits":{"cpu":"200m","memory":"200Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, CPUPolicy: &doRestart, MemPolicy: &doRestart, }, { Name: "c2", - Resources: &ContainerResources{CPUReq: "250m", CPULim: "350m", MemReq: "250Mi", MemLim: "350Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "250m", CPULim: "350m", MemReq: "250Mi", MemLim: "350Mi"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, RestartCount: 1, }, { Name: "c3", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "200m", MemReq: "100Mi", MemLim: "200Mi"}, CPUPolicy: &doRestart, MemPolicy: &doRestart, RestartCount: 1, @@ -1268,10 +831,10 @@ func doPodResizeTests() { }, { name: "Guaranteed QoS pod, one container - increase CPU & memory with an extended resource", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi", + Resources: &e2epod.ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi", ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, @@ -1280,10 +843,10 @@ func doPodResizeTests() { patchString: `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} ]}}`, - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi", + Resources: &e2epod.ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi", ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, CPUPolicy: &noRestart, MemPolicy: &noRestart, @@ -1299,7 +862,9 @@ func doPodResizeTests() { tc := tests[idx] ginkgo.It(tc.name, func(ctx context.Context) { ginkgo.By("check if in place pod vertical scaling is supported", func() { - if !isInPlacePodVerticalScalingSupportedByRuntime(ctx, f.ClientSet) || framework.NodeOSDistroIs("windows") { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + if !e2epod.IsInPlacePodVerticalScalingSupportedByRuntime(node) || framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") } }) @@ -1307,9 +872,9 @@ func doPodResizeTests() { var pErr error tStamp := strconv.Itoa(time.Now().Nanosecond()) - initDefaultResizePolicy(tc.containers) - initDefaultResizePolicy(tc.expected) - testPod = makeTestPod(f.Namespace.Name, "testpod", tStamp, tc.containers) + e2epod.InitDefaultResizePolicy(tc.containers) + e2epod.InitDefaultResizePolicy(tc.expected) + testPod = e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod", tStamp, tc.containers) testPod = e2epod.MustMixinRestrictedPodSecurity(testPod) if tc.addExtendedResource { @@ -1330,46 +895,46 @@ func doPodResizeTests() { newPod := podClient.CreateSync(ctx, testPod) ginkgo.By("verifying initial pod resources, allocations are as expected") - verifyPodResources(newPod, tc.containers) + e2epod.VerifyPodResources(newPod, tc.containers) ginkgo.By("verifying initial pod resize policy is as expected") - verifyPodResizePolicy(newPod, tc.containers) + e2epod.VerifyPodResizePolicy(newPod, tc.containers) ginkgo.By("verifying initial pod status resources are as expected") - verifyPodStatusResources(newPod, tc.containers) + e2epod.VerifyPodStatusResources(newPod, tc.containers) ginkgo.By("verifying initial cgroup config are as expected") - framework.ExpectNoError(verifyPodContainersCgroupValues(ctx, f, newPod, tc.containers)) + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, newPod, tc.containers)) - patchAndVerify := func(patchString string, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, opStr string, isRollback bool) { + patchAndVerify := func(patchString string, expectedContainers []e2epod.ResizableContainerInfo, initialContainers []e2epod.ResizableContainerInfo, opStr string, isRollback bool) { ginkgo.By(fmt.Sprintf("patching pod for %s", opStr)) patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(context.TODO(), newPod.Name, types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}) framework.ExpectNoError(pErr, fmt.Sprintf("failed to patch pod for %s", opStr)) ginkgo.By(fmt.Sprintf("verifying pod patched for %s", opStr)) - verifyPodResources(patchedPod, expectedContainers) - gomega.Eventually(ctx, verifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + e2epod.VerifyPodResources(patchedPod, expectedContainers) + gomega.Eventually(ctx, e2epod.VerifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). WithArguments(patchedPod, initialContainers). Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") ginkgo.By(fmt.Sprintf("waiting for %s to be actuated", opStr)) - resizedPod := waitForPodResizeActuation(ctx, f, podClient, newPod, patchedPod, expectedContainers, initialContainers, isRollback) + resizedPod := e2epod.WaitForPodResizeActuation(ctx, f, podClient, newPod, patchedPod, expectedContainers, initialContainers, isRollback) // Check cgroup values only for containerd versions before 1.6.9 ginkgo.By(fmt.Sprintf("verifying pod container's cgroup values after %s", opStr)) - framework.ExpectNoError(verifyPodContainersCgroupValues(ctx, f, resizedPod, expectedContainers)) + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, resizedPod, expectedContainers)) ginkgo.By(fmt.Sprintf("verifying pod resources after %s", opStr)) - verifyPodResources(resizedPod, expectedContainers) + e2epod.VerifyPodResources(resizedPod, expectedContainers) ginkgo.By(fmt.Sprintf("verifying pod allocations after %s", opStr)) - gomega.Eventually(ctx, verifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + gomega.Eventually(ctx, e2epod.VerifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). WithArguments(resizedPod, expectedContainers). Should(gomega.BeNil(), "failed to verify Pod allocations for resizedPod") } patchAndVerify(tc.patchString, tc.expected, tc.containers, "resize", false) - rbPatchStr, err := genPatchString(tc.containers) + rbPatchStr, err := e2epod.ResizeContainerPatch(tc.containers) framework.ExpectNoError(err) // Resize has been actuated, test rollback patchAndVerify(rbPatchStr, tc.containers, tc.expected, "rollback", true) @@ -1389,16 +954,16 @@ func doPodResizeErrorTests() { type testCase struct { name string - containers []TestContainerInfo + containers []e2epod.ResizableContainerInfo patchString string patchError string - expected []TestContainerInfo + expected []e2epod.ResizableContainerInfo } tests := []testCase{ { name: "BestEffort pod - try requesting memory, expect error", - containers: []TestContainerInfo{ + containers: []e2epod.ResizableContainerInfo{ { Name: "c1", }, @@ -1407,7 +972,7 @@ func doPodResizeErrorTests() { {"name":"c1", "resources":{"requests":{"memory":"400Mi"}}} ]}}`, patchError: "Pod QoS is immutable", - expected: []TestContainerInfo{ + expected: []e2epod.ResizableContainerInfo{ { Name: "c1", }, @@ -1421,7 +986,9 @@ func doPodResizeErrorTests() { tc := tests[idx] ginkgo.It(tc.name, func(ctx context.Context) { ginkgo.By("check if in place pod vertical scaling is supported", func() { - if !isInPlacePodVerticalScalingSupportedByRuntime(ctx, f.ClientSet) || framework.NodeOSDistroIs("windows") { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + if !e2epod.IsInPlacePodVerticalScalingSupportedByRuntime(node) || framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") } }) @@ -1429,20 +996,20 @@ func doPodResizeErrorTests() { var pErr error tStamp := strconv.Itoa(time.Now().Nanosecond()) - initDefaultResizePolicy(tc.containers) - initDefaultResizePolicy(tc.expected) - testPod = makeTestPod(f.Namespace.Name, "testpod", tStamp, tc.containers) + e2epod.InitDefaultResizePolicy(tc.containers) + e2epod.InitDefaultResizePolicy(tc.expected) + testPod = e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod", tStamp, tc.containers) testPod = e2epod.MustMixinRestrictedPodSecurity(testPod) ginkgo.By("creating pod") newPod := podClient.CreateSync(ctx, testPod) ginkgo.By("verifying initial pod resources, allocations, and policy are as expected") - verifyPodResources(newPod, tc.containers) - verifyPodResizePolicy(newPod, tc.containers) + e2epod.VerifyPodResources(newPod, tc.containers) + e2epod.VerifyPodResizePolicy(newPod, tc.containers) ginkgo.By("verifying initial pod status resources and cgroup config are as expected") - verifyPodStatusResources(newPod, tc.containers) + e2epod.VerifyPodStatusResources(newPod, tc.containers) ginkgo.By("patching pod for resize") patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(ctx, newPod.Name, @@ -1455,10 +1022,10 @@ func doPodResizeErrorTests() { } ginkgo.By("verifying pod resources after patch") - verifyPodResources(patchedPod, tc.expected) + e2epod.VerifyPodResources(patchedPod, tc.expected) ginkgo.By("verifying pod allocations after patch") - gomega.Eventually(ctx, verifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + gomega.Eventually(ctx, e2epod.VerifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). WithArguments(patchedPod, tc.expected). Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") diff --git a/test/e2e/framework/node/helper.go b/test/e2e/framework/node/helper.go index 9bb76d97159..59824a0d4bf 100644 --- a/test/e2e/framework/node/helper.go +++ b/test/e2e/framework/node/helper.go @@ -23,14 +23,13 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - testutils "k8s.io/kubernetes/test/utils" "k8s.io/kubernetes/test/e2e/framework" + testutils "k8s.io/kubernetes/test/utils" ) const ( @@ -166,3 +165,13 @@ func taintExists(taints []v1.Taint, taintToFind *v1.Taint) bool { } return false } + +// IsARM64 checks whether the k8s Node has arm64 arch. +func IsARM64(node *v1.Node) bool { + arch, ok := node.Labels["kubernetes.io/arch"] + if ok { + return arch == "arm64" + } + + return false +} diff --git a/test/e2e/framework/pod/resize.go b/test/e2e/framework/pod/resize.go new file mode 100644 index 00000000000..f7b8cb54b88 --- /dev/null +++ b/test/e2e/framework/pod/resize.go @@ -0,0 +1,470 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + + semver "github.com/blang/semver/v4" + "github.com/google/go-cmp/cmp" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + podutil "k8s.io/kubernetes/pkg/api/v1/pod" + kubecm "k8s.io/kubernetes/pkg/kubelet/cm" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" +) + +const ( + CgroupCPUPeriod string = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" + CgroupCPUShares string = "/sys/fs/cgroup/cpu/cpu.shares" + CgroupCPUQuota string = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" + CgroupMemLimit string = "/sys/fs/cgroup/memory/memory.limit_in_bytes" + Cgroupv2MemLimit string = "/sys/fs/cgroup/memory.max" + Cgroupv2MemRequest string = "/sys/fs/cgroup/memory.min" + Cgroupv2CPULimit string = "/sys/fs/cgroup/cpu.max" + Cgroupv2CPURequest string = "/sys/fs/cgroup/cpu.weight" + CPUPeriod string = "100000" + MinContainerRuntimeVersion string = "1.6.9" +) + +var ( + podOnCgroupv2Node *bool +) + +type ContainerResources struct { + CPUReq string + CPULim string + MemReq string + MemLim string + EphStorReq string + EphStorLim string + ExtendedResourceReq string + ExtendedResourceLim string +} + +type ContainerAllocations struct { + CPUAlloc string + MemAlloc string + ephStorAlloc string + ExtendedResourceAlloc string +} + +type ResizableContainerInfo struct { + Name string + Resources *ContainerResources + Allocations *ContainerAllocations + CPUPolicy *v1.ResourceResizeRestartPolicy + MemPolicy *v1.ResourceResizeRestartPolicy + RestartCount int32 +} + +type containerPatch struct { + Name string `json:"name"` + Resources struct { + Requests struct { + CPU string `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` + EphStor string `json:"ephemeral-storage,omitempty"` + } `json:"requests"` + Limits struct { + CPU string `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` + EphStor string `json:"ephemeral-storage,omitempty"` + } `json:"limits"` + } `json:"resources"` +} + +type patchSpec struct { + Spec struct { + Containers []containerPatch `json:"containers"` + } `json:"spec"` +} + +func IsInPlacePodVerticalScalingSupportedByRuntime(node *v1.Node) bool { + re := regexp.MustCompile("containerd://(.*)") + match := re.FindStringSubmatch(node.Status.NodeInfo.ContainerRuntimeVersion) + if len(match) != 2 { + return false + } + if ver, verr := semver.ParseTolerant(match[1]); verr == nil { + if ver.Compare(semver.MustParse(MinContainerRuntimeVersion)) < 0 { + return false + } + return true + } + return false +} + +func getTestResourceInfo(tcInfo ResizableContainerInfo) (v1.ResourceRequirements, v1.ResourceList, []v1.ContainerResizePolicy) { + var res v1.ResourceRequirements + var alloc v1.ResourceList + var resizePol []v1.ContainerResizePolicy + + if tcInfo.Resources != nil { + var lim, req v1.ResourceList + if tcInfo.Resources.CPULim != "" || tcInfo.Resources.MemLim != "" || tcInfo.Resources.EphStorLim != "" { + lim = make(v1.ResourceList) + } + if tcInfo.Resources.CPUReq != "" || tcInfo.Resources.MemReq != "" || tcInfo.Resources.EphStorReq != "" { + req = make(v1.ResourceList) + } + if tcInfo.Resources.CPULim != "" { + lim[v1.ResourceCPU] = resource.MustParse(tcInfo.Resources.CPULim) + } + if tcInfo.Resources.MemLim != "" { + lim[v1.ResourceMemory] = resource.MustParse(tcInfo.Resources.MemLim) + } + if tcInfo.Resources.EphStorLim != "" { + lim[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Resources.EphStorLim) + } + if tcInfo.Resources.CPUReq != "" { + req[v1.ResourceCPU] = resource.MustParse(tcInfo.Resources.CPUReq) + } + if tcInfo.Resources.MemReq != "" { + req[v1.ResourceMemory] = resource.MustParse(tcInfo.Resources.MemReq) + } + if tcInfo.Resources.EphStorReq != "" { + req[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Resources.EphStorReq) + } + res = v1.ResourceRequirements{Limits: lim, Requests: req} + } + if tcInfo.Allocations != nil { + alloc = make(v1.ResourceList) + if tcInfo.Allocations.CPUAlloc != "" { + alloc[v1.ResourceCPU] = resource.MustParse(tcInfo.Allocations.CPUAlloc) + } + if tcInfo.Allocations.MemAlloc != "" { + alloc[v1.ResourceMemory] = resource.MustParse(tcInfo.Allocations.MemAlloc) + } + if tcInfo.Allocations.ephStorAlloc != "" { + alloc[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Allocations.ephStorAlloc) + } + + } + if tcInfo.CPUPolicy != nil { + cpuPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceCPU, RestartPolicy: *tcInfo.CPUPolicy} + resizePol = append(resizePol, cpuPol) + } + if tcInfo.MemPolicy != nil { + memPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceMemory, RestartPolicy: *tcInfo.MemPolicy} + resizePol = append(resizePol, memPol) + } + return res, alloc, resizePol +} + +func InitDefaultResizePolicy(containers []ResizableContainerInfo) { + noRestart := v1.NotRequired + setDefaultPolicy := func(ci *ResizableContainerInfo) { + if ci.CPUPolicy == nil { + ci.CPUPolicy = &noRestart + } + if ci.MemPolicy == nil { + ci.MemPolicy = &noRestart + } + } + for i := range containers { + setDefaultPolicy(&containers[i]) + } +} + +func makeResizableContainer(tcInfo ResizableContainerInfo) (v1.Container, v1.ContainerStatus) { + cmd := "grep Cpus_allowed_list /proc/self/status | cut -f2 && sleep 1d" + res, alloc, resizePol := getTestResourceInfo(tcInfo) + + tc := v1.Container{ + Name: tcInfo.Name, + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"/bin/sh"}, + Args: []string{"-c", cmd}, + Resources: res, + ResizePolicy: resizePol, + } + + tcStatus := v1.ContainerStatus{ + Name: tcInfo.Name, + AllocatedResources: alloc, + } + return tc, tcStatus +} + +func MakePodWithResizableContainers(ns, name, timeStamp string, tcInfo []ResizableContainerInfo) *v1.Pod { + var testContainers []v1.Container + + for _, ci := range tcInfo { + tc, _ := makeResizableContainer(ci) + testContainers = append(testContainers, tc) + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{ + "time": timeStamp, + }, + }, + Spec: v1.PodSpec{ + OS: &v1.PodOS{Name: v1.Linux}, + Containers: testContainers, + RestartPolicy: v1.RestartPolicyOnFailure, + }, + } + return pod +} + +func VerifyPodResizePolicy(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) { + ginkgo.GinkgoHelper() + for i, wantCtr := range wantCtrs { + gotCtr := &gotPod.Spec.Containers[i] + ctr, _ := makeResizableContainer(wantCtr) + gomega.Expect(gotCtr.Name).To(gomega.Equal(ctr.Name)) + gomega.Expect(gotCtr.ResizePolicy).To(gomega.Equal(ctr.ResizePolicy)) + } +} + +func VerifyPodResources(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) { + ginkgo.GinkgoHelper() + for i, wantCtr := range wantCtrs { + gotCtr := &gotPod.Spec.Containers[i] + ctr, _ := makeResizableContainer(wantCtr) + gomega.Expect(gotCtr.Name).To(gomega.Equal(ctr.Name)) + gomega.Expect(gotCtr.Resources).To(gomega.Equal(ctr.Resources)) + } +} + +func VerifyPodAllocations(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) error { + ginkgo.GinkgoHelper() + for i, wantCtr := range wantCtrs { + gotCtrStatus := &gotPod.Status.ContainerStatuses[i] + if wantCtr.Allocations == nil { + if wantCtr.Resources != nil { + alloc := &ContainerAllocations{CPUAlloc: wantCtr.Resources.CPUReq, MemAlloc: wantCtr.Resources.MemReq} + wantCtr.Allocations = alloc + defer func() { + wantCtr.Allocations = nil + }() + } + } + + _, ctrStatus := makeResizableContainer(wantCtr) + gomega.Expect(gotCtrStatus.Name).To(gomega.Equal(ctrStatus.Name)) + if !cmp.Equal(gotCtrStatus.AllocatedResources, ctrStatus.AllocatedResources) { + return fmt.Errorf("failed to verify Pod allocations, allocated resources not equal to expected") + } + } + return nil +} + +func VerifyPodStatusResources(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) { + ginkgo.GinkgoHelper() + for i, wantCtr := range wantCtrs { + gotCtrStatus := &gotPod.Status.ContainerStatuses[i] + ctr, _ := makeResizableContainer(wantCtr) + gomega.Expect(gotCtrStatus.Name).To(gomega.Equal(ctr.Name)) + gomega.Expect(ctr.Resources).To(gomega.Equal(*gotCtrStatus.Resources)) + } +} + +func isPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) bool { + // Determine if pod is running on cgroupv2 or cgroupv1 node + //TODO(vinaykul,InPlacePodVerticalScaling): Is there a better way to determine this? + cmd := "mount -t cgroup2" + out, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd) + if err != nil { + return false + } + return len(out) != 0 +} + +func VerifyPodContainersCgroupValues(ctx context.Context, f *framework.Framework, pod *v1.Pod, tcInfo []ResizableContainerInfo) error { + ginkgo.GinkgoHelper() + if podOnCgroupv2Node == nil { + value := isPodOnCgroupv2Node(f, pod) + podOnCgroupv2Node = &value + } + cgroupMemLimit := Cgroupv2MemLimit + cgroupCPULimit := Cgroupv2CPULimit + cgroupCPURequest := Cgroupv2CPURequest + if !*podOnCgroupv2Node { + cgroupMemLimit = CgroupMemLimit + cgroupCPULimit = CgroupCPUQuota + cgroupCPURequest = CgroupCPUShares + } + verifyCgroupValue := func(cName, cgPath, expectedCgValue string) error { + cmd := fmt.Sprintf("head -n 1 %s", cgPath) + framework.Logf("Namespace %s Pod %s Container %s - looking for cgroup value %s in path %s", + pod.Namespace, pod.Name, cName, expectedCgValue, cgPath) + cgValue, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, cName, "/bin/sh", "-c", cmd) + if err != nil { + return fmt.Errorf("failed to find expected value %q in container cgroup %q", expectedCgValue, cgPath) + } + cgValue = strings.Trim(cgValue, "\n") + if cgValue != expectedCgValue { + return fmt.Errorf("cgroup value %q not equal to expected %q", cgValue, expectedCgValue) + } + return nil + } + for _, ci := range tcInfo { + if ci.Resources == nil { + continue + } + tc, _ := makeResizableContainer(ci) + if tc.Resources.Limits != nil || tc.Resources.Requests != nil { + var expectedCPUShares int64 + var expectedCPULimitString, expectedMemLimitString string + expectedMemLimitInBytes := tc.Resources.Limits.Memory().Value() + cpuRequest := tc.Resources.Requests.Cpu() + cpuLimit := tc.Resources.Limits.Cpu() + if cpuRequest.IsZero() && !cpuLimit.IsZero() { + expectedCPUShares = int64(kubecm.MilliCPUToShares(cpuLimit.MilliValue())) + } else { + expectedCPUShares = int64(kubecm.MilliCPUToShares(cpuRequest.MilliValue())) + } + cpuQuota := kubecm.MilliCPUToQuota(cpuLimit.MilliValue(), kubecm.QuotaPeriod) + if cpuLimit.IsZero() { + cpuQuota = -1 + } + expectedCPULimitString = strconv.FormatInt(cpuQuota, 10) + expectedMemLimitString = strconv.FormatInt(expectedMemLimitInBytes, 10) + if *podOnCgroupv2Node { + if expectedCPULimitString == "-1" { + expectedCPULimitString = "max" + } + expectedCPULimitString = fmt.Sprintf("%s %s", expectedCPULimitString, CPUPeriod) + if expectedMemLimitString == "0" { + expectedMemLimitString = "max" + } + // convert cgroup v1 cpu.shares value to cgroup v2 cpu.weight value + // https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2 + expectedCPUShares = int64(1 + ((expectedCPUShares-2)*9999)/262142) + } + if expectedMemLimitString != "0" { + err := verifyCgroupValue(ci.Name, cgroupMemLimit, expectedMemLimitString) + if err != nil { + return err + } + } + err := verifyCgroupValue(ci.Name, cgroupCPULimit, expectedCPULimitString) + if err != nil { + return err + } + err = verifyCgroupValue(ci.Name, cgroupCPURequest, strconv.FormatInt(expectedCPUShares, 10)) + if err != nil { + return err + } + } + } + return nil +} + +func waitForContainerRestart(ctx context.Context, podClient *PodClient, pod *v1.Pod, expectedContainers []ResizableContainerInfo, initialContainers []ResizableContainerInfo, isRollback bool) error { + ginkgo.GinkgoHelper() + var restartContainersExpected []string + + restartContainers := expectedContainers + // if we're rolling back, extract restart counts from test case "expected" containers + if isRollback { + restartContainers = initialContainers + } + + for _, ci := range restartContainers { + if ci.RestartCount > 0 { + restartContainersExpected = append(restartContainersExpected, ci.Name) + } + } + if len(restartContainersExpected) == 0 { + return nil + } + + pod, err := podClient.Get(ctx, pod.Name, metav1.GetOptions{}) + if err != nil { + return err + } + restartedContainersCount := 0 + for _, cName := range restartContainersExpected { + cs, _ := podutil.GetContainerStatus(pod.Status.ContainerStatuses, cName) + if cs.RestartCount < 1 { + break + } + restartedContainersCount++ + } + if restartedContainersCount == len(restartContainersExpected) { + return nil + } + if restartedContainersCount > len(restartContainersExpected) { + return fmt.Errorf("more container restarts than expected") + } else { + return fmt.Errorf("less container restarts than expected") + } +} + +func WaitForPodResizeActuation(ctx context.Context, f *framework.Framework, podClient *PodClient, pod, patchedPod *v1.Pod, expectedContainers []ResizableContainerInfo, initialContainers []ResizableContainerInfo, isRollback bool) *v1.Pod { + ginkgo.GinkgoHelper() + var resizedPod *v1.Pod + var pErr error + timeouts := framework.NewTimeoutContext() + // Wait for container restart + gomega.Eventually(ctx, waitForContainerRestart, timeouts.PodStartShort, timeouts.Poll). + WithArguments(podClient, pod, expectedContainers, initialContainers, isRollback). + ShouldNot(gomega.HaveOccurred(), "failed waiting for expected container restart") + // Verify Pod Containers Cgroup Values + gomega.Eventually(ctx, VerifyPodContainersCgroupValues, timeouts.PodStartShort, timeouts.Poll). + WithArguments(f, patchedPod, expectedContainers). + ShouldNot(gomega.HaveOccurred(), "failed to verify container cgroup values to match expected") + // Wait for pod resource allocations to equal expected values after resize + gomega.Eventually(ctx, func() error { + resizedPod, pErr = podClient.Get(ctx, pod.Name, metav1.GetOptions{}) + if pErr != nil { + return pErr + } + return VerifyPodAllocations(resizedPod, expectedContainers) + }, timeouts.PodStartShort, timeouts.Poll). + ShouldNot(gomega.HaveOccurred(), "timed out waiting for pod resource allocation values to match expected") + return resizedPod +} + +// ResizeContainerPatch generates a patch string to resize the pod container. +func ResizeContainerPatch(containers []ResizableContainerInfo) (string, error) { + var patch patchSpec + + for _, container := range containers { + var cPatch containerPatch + cPatch.Name = container.Name + cPatch.Resources.Requests.CPU = container.Resources.CPUReq + cPatch.Resources.Requests.Memory = container.Resources.MemReq + cPatch.Resources.Limits.CPU = container.Resources.CPULim + cPatch.Resources.Limits.Memory = container.Resources.MemLim + + patch.Spec.Containers = append(patch.Spec.Containers, cPatch) + } + + patchBytes, err := json.Marshal(patch) + if err != nil { + return "", err + } + + return string(patchBytes), nil +} diff --git a/test/e2e/node/pod_resize.go b/test/e2e/node/pod_resize.go index 9a087b5760b..81c45bef154 100644 --- a/test/e2e/node/pod_resize.go +++ b/test/e2e/node/pod_resize.go @@ -18,576 +18,41 @@ package node import ( "context" - "encoding/json" "fmt" - "regexp" - "runtime" "strconv" - "strings" "time" - semver "github.com/blang/semver/v4" - "github.com/google/go-cmp/cmp" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - clientset "k8s.io/client-go/kubernetes" - podutil "k8s.io/kubernetes/pkg/api/v1/pod" resourceapi "k8s.io/kubernetes/pkg/api/v1/resource" - kubecm "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" - e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" - imageutils "k8s.io/kubernetes/test/utils/image" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" ) -const ( - CgroupCPUPeriod string = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" - CgroupCPUShares string = "/sys/fs/cgroup/cpu/cpu.shares" - CgroupCPUQuota string = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" - CgroupMemLimit string = "/sys/fs/cgroup/memory/memory.limit_in_bytes" - Cgroupv2MemLimit string = "/sys/fs/cgroup/memory.max" - Cgroupv2MemRequest string = "/sys/fs/cgroup/memory.min" - Cgroupv2CPULimit string = "/sys/fs/cgroup/cpu.max" - Cgroupv2CPURequest string = "/sys/fs/cgroup/cpu.weight" - CpuPeriod string = "100000" - - PollInterval time.Duration = 2 * time.Second - PollTimeout time.Duration = 4 * time.Minute - - fakeExtendedResource = "dummy.com/dummy" -) - -type ContainerResources struct { - CPUReq, CPULim, MemReq, MemLim, EphStorReq, EphStorLim, ExtendedResourceReq, ExtendedResourceLim string -} - -type ContainerAllocations struct { - CPUAlloc, MemAlloc, ephStorAlloc, ExtendedResourceAlloc string -} - -type TestContainerInfo struct { - Name string - Resources *ContainerResources - Allocations *ContainerAllocations - CPUPolicy *v1.ResourceResizeRestartPolicy - MemPolicy *v1.ResourceResizeRestartPolicy - RestartCount int32 -} - -type containerPatch struct { - Name string `json:"name"` - Resources struct { - Requests struct { - CPU string `json:"cpu,omitempty"` - Memory string `json:"memory,omitempty"` - EphStor string `json:"ephemeral-storage,omitempty"` - } `json:"requests"` - Limits struct { - CPU string `json:"cpu,omitempty"` - Memory string `json:"memory,omitempty"` - EphStor string `json:"ephemeral-storage,omitempty"` - } `json:"limits"` - } `json:"resources"` -} - -type patchSpec struct { - Spec struct { - Containers []containerPatch `json:"containers"` - } `json:"spec"` -} - -func isInPlaceResizeSupportedByRuntime(c clientset.Interface, nodeName string) bool { - //TODO(vinaykul,InPlacePodVerticalScaling): Can we optimize this? - node, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - if err != nil { - return false - } - re := regexp.MustCompile("containerd://(.*)") - match := re.FindStringSubmatch(node.Status.NodeInfo.ContainerRuntimeVersion) - if len(match) != 2 { - return false - } - if ver, verr := semver.ParseTolerant(match[1]); verr == nil { - if ver.Compare(semver.MustParse("1.6.9")) < 0 { - return false - } - return true - } - return false -} - -func getTestResourceInfo(tcInfo TestContainerInfo) (v1.ResourceRequirements, v1.ResourceList, []v1.ContainerResizePolicy) { - var res v1.ResourceRequirements - var alloc v1.ResourceList - var resizePol []v1.ContainerResizePolicy - - if tcInfo.Resources != nil { - var lim, req v1.ResourceList - if tcInfo.Resources.CPULim != "" || tcInfo.Resources.MemLim != "" || tcInfo.Resources.EphStorLim != "" { - lim = make(v1.ResourceList) - } - if tcInfo.Resources.CPUReq != "" || tcInfo.Resources.MemReq != "" || tcInfo.Resources.EphStorReq != "" { - req = make(v1.ResourceList) - } - if tcInfo.Resources.CPULim != "" { - lim[v1.ResourceCPU] = resource.MustParse(tcInfo.Resources.CPULim) - } - if tcInfo.Resources.MemLim != "" { - lim[v1.ResourceMemory] = resource.MustParse(tcInfo.Resources.MemLim) - } - if tcInfo.Resources.EphStorLim != "" { - lim[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Resources.EphStorLim) - } - if tcInfo.Resources.ExtendedResourceLim != "" { - lim[fakeExtendedResource] = resource.MustParse(tcInfo.Resources.ExtendedResourceLim) - } - if tcInfo.Resources.CPUReq != "" { - req[v1.ResourceCPU] = resource.MustParse(tcInfo.Resources.CPUReq) - } - if tcInfo.Resources.MemReq != "" { - req[v1.ResourceMemory] = resource.MustParse(tcInfo.Resources.MemReq) - } - if tcInfo.Resources.EphStorReq != "" { - req[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Resources.EphStorReq) - } - if tcInfo.Resources.ExtendedResourceReq != "" { - req[fakeExtendedResource] = resource.MustParse(tcInfo.Resources.ExtendedResourceReq) - } - res = v1.ResourceRequirements{Limits: lim, Requests: req} - } - if tcInfo.Allocations != nil { - alloc = make(v1.ResourceList) - if tcInfo.Allocations.CPUAlloc != "" { - alloc[v1.ResourceCPU] = resource.MustParse(tcInfo.Allocations.CPUAlloc) - } - if tcInfo.Allocations.MemAlloc != "" { - alloc[v1.ResourceMemory] = resource.MustParse(tcInfo.Allocations.MemAlloc) - } - if tcInfo.Allocations.ephStorAlloc != "" { - alloc[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Allocations.ephStorAlloc) - } - if tcInfo.Allocations.ExtendedResourceAlloc != "" { - alloc[fakeExtendedResource] = resource.MustParse(tcInfo.Allocations.ExtendedResourceAlloc) - } - } - if tcInfo.CPUPolicy != nil { - cpuPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceCPU, RestartPolicy: *tcInfo.CPUPolicy} - resizePol = append(resizePol, cpuPol) - } - if tcInfo.MemPolicy != nil { - memPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceMemory, RestartPolicy: *tcInfo.MemPolicy} - resizePol = append(resizePol, memPol) - } - return res, alloc, resizePol -} - -func initDefaultResizePolicy(containers []TestContainerInfo) { - noRestart := v1.NotRequired - setDefaultPolicy := func(ci *TestContainerInfo) { - if ci.CPUPolicy == nil { - ci.CPUPolicy = &noRestart - } - if ci.MemPolicy == nil { - ci.MemPolicy = &noRestart - } - } - for i := range containers { - setDefaultPolicy(&containers[i]) - } -} - -func makeTestContainer(tcInfo TestContainerInfo) (v1.Container, v1.ContainerStatus) { - res, alloc, resizePol := getTestResourceInfo(tcInfo) - bTrue := true - bFalse := false - userID := int64(1001) - userName := "ContainerUser" - - var securityContext *v1.SecurityContext - - if framework.NodeOSDistroIs("windows") { - securityContext = &v1.SecurityContext{ - RunAsNonRoot: &bTrue, - WindowsOptions: &v1.WindowsSecurityContextOptions{ - RunAsUserName: &userName, - }, - } - } else { - securityContext = &v1.SecurityContext{ - Privileged: &bFalse, - AllowPrivilegeEscalation: &bFalse, - RunAsUser: &userID, - RunAsNonRoot: &bTrue, - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{"ALL"}, - }, - SeccompProfile: &v1.SeccompProfile{ - Type: v1.SeccompProfileTypeRuntimeDefault, - }, - } - } - - tc := v1.Container{ - Name: tcInfo.Name, - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"/bin/sh"}, - Args: []string{"-c", e2epod.InfiniteSleepCommand}, - Resources: res, - ResizePolicy: resizePol, - SecurityContext: securityContext, - } - - tcStatus := v1.ContainerStatus{ - Name: tcInfo.Name, - AllocatedResources: alloc, - } - return tc, tcStatus -} - -func makeTestPod(ns, name, timeStamp string, tcInfo []TestContainerInfo) *v1.Pod { - var testContainers []v1.Container - var podOS *v1.PodOS - - for _, ci := range tcInfo { - tc, _ := makeTestContainer(ci) - testContainers = append(testContainers, tc) - } - - if framework.NodeOSDistroIs("windows") { - podOS = &v1.PodOS{Name: v1.OSName("windows")} - } else { - podOS = &v1.PodOS{Name: v1.OSName(runtime.GOOS)} - } - - pod := &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - Labels: map[string]string{ - "name": "fooPod", - "time": timeStamp, - }, - }, - Spec: v1.PodSpec{ - OS: podOS, - Containers: testContainers, - RestartPolicy: v1.RestartPolicyOnFailure, - }, - } - return pod -} - -func verifyPodResizePolicy(pod *v1.Pod, tcInfo []TestContainerInfo) { - cMap := make(map[string]*v1.Container) - for i, c := range pod.Spec.Containers { - cMap[c.Name] = &pod.Spec.Containers[i] - } - for _, ci := range tcInfo { - gomega.Expect(cMap).Should(gomega.HaveKey(ci.Name)) - c := cMap[ci.Name] - tc, _ := makeTestContainer(ci) - gomega.Expect(tc.ResizePolicy).To(gomega.Equal(c.ResizePolicy)) - } -} - -func verifyPodResources(pod *v1.Pod, tcInfo []TestContainerInfo) { - cMap := make(map[string]*v1.Container) - for i, c := range pod.Spec.Containers { - cMap[c.Name] = &pod.Spec.Containers[i] - } - for _, ci := range tcInfo { - gomega.Expect(cMap).Should(gomega.HaveKey(ci.Name)) - c := cMap[ci.Name] - tc, _ := makeTestContainer(ci) - gomega.Expect(tc.Resources).To(gomega.Equal(c.Resources)) - } -} - -func verifyPodAllocations(pod *v1.Pod, tcInfo []TestContainerInfo, flagError bool) bool { - cStatusMap := make(map[string]*v1.ContainerStatus) - for i, c := range pod.Status.ContainerStatuses { - cStatusMap[c.Name] = &pod.Status.ContainerStatuses[i] - } - - for _, ci := range tcInfo { - gomega.Expect(cStatusMap).Should(gomega.HaveKey(ci.Name)) - cStatus := cStatusMap[ci.Name] - if ci.Allocations == nil { - if ci.Resources != nil { - alloc := &ContainerAllocations{CPUAlloc: ci.Resources.CPUReq, MemAlloc: ci.Resources.MemReq, - ExtendedResourceAlloc: ci.Resources.ExtendedResourceReq} - ci.Allocations = alloc - defer func() { - ci.Allocations = nil - }() - } - } - - _, tcStatus := makeTestContainer(ci) - if flagError { - gomega.Expect(tcStatus.AllocatedResources).To(gomega.Equal(cStatus.AllocatedResources)) - } - if !cmp.Equal(cStatus.AllocatedResources, tcStatus.AllocatedResources) { - return false - } - } - return true -} - -func verifyPodStatusResources(pod *v1.Pod, tcInfo []TestContainerInfo) { - csMap := make(map[string]*v1.ContainerStatus) - for i, c := range pod.Status.ContainerStatuses { - csMap[c.Name] = &pod.Status.ContainerStatuses[i] - } - for _, ci := range tcInfo { - gomega.Expect(csMap).Should(gomega.HaveKey(ci.Name)) - cs := csMap[ci.Name] - tc, _ := makeTestContainer(ci) - gomega.Expect(tc.Resources).To(gomega.Equal(*cs.Resources)) - //gomega.Expect(cs.RestartCount).To(gomega.Equal(ci.RestartCount)) - } -} - -func isPodOnCgroupv2Node(pod *v1.Pod) bool { - // Determine if pod is running on cgroupv2 or cgroupv1 node - //TODO(vinaykul,InPlacePodVerticalScaling): Is there a better way to determine this? - cgroupv2File := "/sys/fs/cgroup/cgroup.controllers" - _, err := e2ekubectl.RunKubectl(pod.Namespace, "exec", pod.Name, "--", "ls", cgroupv2File) - if err == nil { - return true - } - return false -} - -func verifyPodContainersCgroupValues(pod *v1.Pod, tcInfo []TestContainerInfo, flagError bool) bool { - podOnCgroupv2Node := isPodOnCgroupv2Node(pod) - cgroupMemLimit := Cgroupv2MemLimit - cgroupCPULimit := Cgroupv2CPULimit - cgroupCPURequest := Cgroupv2CPURequest - if !podOnCgroupv2Node { - cgroupMemLimit = CgroupMemLimit - cgroupCPULimit = CgroupCPUQuota - cgroupCPURequest = CgroupCPUShares - } - verifyCgroupValue := func(cName, cgPath, expectedCgValue string) bool { - cmd := []string{"head", "-n", "1", cgPath} - framework.Logf("Namespace %s Pod %s Container %s - looking for cgroup value %s in path %s", - pod.Namespace, pod.Name, cName, expectedCgValue, cgPath) - cgValue, err := e2epodoutput.LookForStringInPodExecToContainer(pod.Namespace, pod.Name, cName, cmd, expectedCgValue, PollTimeout) - if flagError { - framework.ExpectNoError(err, fmt.Sprintf("failed to find expected value '%s' in container cgroup '%s'", - expectedCgValue, cgPath)) - } - cgValue = strings.Trim(cgValue, "\n") - if flagError { - gomega.Expect(cgValue).Should(gomega.Equal(expectedCgValue), "cgroup value") - } - if cgValue != expectedCgValue { - return false - } - return true - } - for _, ci := range tcInfo { - if ci.Resources == nil { - continue - } - tc, _ := makeTestContainer(ci) - if tc.Resources.Limits != nil || tc.Resources.Requests != nil { - var cpuShares int64 - var cpuLimitString, memLimitString string - memLimitInBytes := tc.Resources.Limits.Memory().Value() - cpuRequest := tc.Resources.Requests.Cpu() - cpuLimit := tc.Resources.Limits.Cpu() - if cpuRequest.IsZero() && !cpuLimit.IsZero() { - cpuShares = int64(kubecm.MilliCPUToShares(cpuLimit.MilliValue())) - } else { - cpuShares = int64(kubecm.MilliCPUToShares(cpuRequest.MilliValue())) - } - cpuQuota := kubecm.MilliCPUToQuota(cpuLimit.MilliValue(), kubecm.QuotaPeriod) - if cpuLimit.IsZero() { - cpuQuota = -1 - } - cpuLimitString = strconv.FormatInt(cpuQuota, 10) - if podOnCgroupv2Node { - if cpuLimitString == "-1" { - cpuLimitString = "max" - } - cpuLimitString = fmt.Sprintf("%s %s", cpuLimitString, CpuPeriod) - } - memLimitString = strconv.FormatInt(memLimitInBytes, 10) - if podOnCgroupv2Node && memLimitString == "0" { - memLimitString = "max" - } - if memLimitString != "0" { - if !verifyCgroupValue(ci.Name, cgroupMemLimit, memLimitString) { - return false - } - } - if !verifyCgroupValue(ci.Name, cgroupCPULimit, cpuLimitString) { - return false - } - if podOnCgroupv2Node { - // convert cgroup v1 cpu.shares value to cgroup v2 cpu.weight value - cpuShares = int64(1 + ((cpuShares-2)*9999)/262142) - } - if !verifyCgroupValue(ci.Name, cgroupCPURequest, strconv.FormatInt(cpuShares, 10)) { - return false - } - } - } - return true -} - -func waitForPodResizeActuation(c clientset.Interface, podClient *e2epod.PodClient, pod, patchedPod *v1.Pod, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, isRollback bool) *v1.Pod { - - waitForContainerRestart := func() error { - var restartContainersExpected []string - - restartContainers := expectedContainers - // if we're rolling back, extract restart counts from test case "expected" containers - if isRollback { - restartContainers = initialContainers - } - - for _, ci := range restartContainers { - if ci.RestartCount > 0 { - restartContainersExpected = append(restartContainersExpected, ci.Name) - } - } - if len(restartContainersExpected) == 0 { - return nil - } - for start := time.Now(); time.Since(start) < PollTimeout; time.Sleep(PollInterval) { - pod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) - if err != nil { - return err - } - restartedContainersCount := 0 - for _, cName := range restartContainersExpected { - cs, _ := podutil.GetContainerStatus(pod.Status.ContainerStatuses, cName) - expectedRestarts := int32(1) - // if we're rolling back, we should have 2 container restarts - if isRollback { - expectedRestarts = int32(2) - } - if cs.RestartCount < expectedRestarts { - break - } - restartedContainersCount++ - } - if restartedContainersCount == len(restartContainersExpected) { - return nil - } - } - return fmt.Errorf("timed out waiting for expected container restart") - } - waitPodAllocationsEqualsExpected := func() (*v1.Pod, error) { - for start := time.Now(); time.Since(start) < PollTimeout; time.Sleep(PollInterval) { - pod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - if !verifyPodAllocations(pod, expectedContainers, false) { - continue - } - return pod, nil - } - return nil, fmt.Errorf("timed out waiting for pod resource allocation values to match expected") - } - waitContainerCgroupValuesEqualsExpected := func() error { - for start := time.Now(); time.Since(start) < PollTimeout; time.Sleep(PollInterval) { - if !verifyPodContainersCgroupValues(patchedPod, expectedContainers, false) { - continue - } - return nil - } - return fmt.Errorf("timed out waiting for container cgroup values to match expected") - } - waitPodStatusResourcesEqualSpecResources := func() (*v1.Pod, error) { - for start := time.Now(); time.Since(start) < PollTimeout; time.Sleep(PollInterval) { - pod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - differs := false - for idx, c := range pod.Spec.Containers { - if !cmp.Equal(c.Resources, *pod.Status.ContainerStatuses[idx].Resources) { - differs = true - break - } - } - if differs { - continue - } - return pod, nil - } - return nil, fmt.Errorf("timed out waiting for pod spec resources to match pod status resources") - } - rsErr := waitForContainerRestart() - framework.ExpectNoError(rsErr, "failed waiting for expected container restart") - // Wait for pod resource allocations to equal expected values after resize - resizedPod, aErr := waitPodAllocationsEqualsExpected() - framework.ExpectNoError(aErr, "failed to verify pod resource allocation values equals expected values") - //TODO(vinaykul,InPlacePodVerticalScaling): Remove this check once base-OS updates to containerd>=1.6.9 - // containerd needs to add CRI support before Beta (See Node KEP #2273) - if !isInPlaceResizeSupportedByRuntime(c, pod.Spec.NodeName) { - // Wait for PodSpec container resources to equal PodStatus container resources indicating resize is complete - rPod, rErr := waitPodStatusResourcesEqualSpecResources() - framework.ExpectNoError(rErr, "failed to verify pod spec resources equals pod status resources") - - ginkgo.By("verifying pod status after resize") - verifyPodStatusResources(rPod, expectedContainers) - } else if !framework.NodeOSDistroIs("windows") { - // Wait for container cgroup values to equal expected cgroup values after resize - // only for containerd versions before 1.6.9 - cErr := waitContainerCgroupValuesEqualsExpected() - framework.ExpectNoError(cErr, "failed to verify container cgroup values equals expected values") - } - return resizedPod -} - -func genPatchString(containers []TestContainerInfo) (string, error) { - var patch patchSpec - - for _, container := range containers { - var cPatch containerPatch - cPatch.Name = container.Name - cPatch.Resources.Requests.CPU = container.Resources.CPUReq - cPatch.Resources.Requests.Memory = container.Resources.MemReq - cPatch.Resources.Limits.CPU = container.Resources.CPULim - cPatch.Resources.Limits.Memory = container.Resources.MemLim - - patch.Spec.Containers = append(patch.Spec.Containers, cPatch) - } - - patchBytes, err := json.Marshal(patch) - if err != nil { - return "", err - } - - return string(patchBytes), nil -} - func doPodResizeResourceQuotaTests() { f := framework.NewDefaultFramework("pod-resize-resource-quota") var podClient *e2epod.PodClient ginkgo.BeforeEach(func() { podClient = e2epod.NewPodClient(f) }) + timeouts := framework.NewTimeoutContext() ginkgo.It("pod-resize-resource-quota-test", func(ctx context.Context) { + ginkgo.By("check if in place pod vertical scaling is supported", func() { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + if !e2epod.IsInPlacePodVerticalScalingSupportedByRuntime(node) || framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + }) resourceQuota := v1.ResourceQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "resize-resource-quota", @@ -600,19 +65,19 @@ func doPodResizeResourceQuotaTests() { }, }, } - containers := []TestContainerInfo{ + containers := []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"}, }, } patchString := `{"spec":{"containers":[ {"name":"c1", "resources":{"requests":{"cpu":"400m","memory":"400Mi"},"limits":{"cpu":"400m","memory":"400Mi"}}} ]}}` - expected := []TestContainerInfo{ + expected := []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "400Mi", MemLim: "400Mi"}, + Resources: &e2epod.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "400Mi", MemLim: "400Mi"}, }, } patchStringExceedCPU := `{"spec":{"containers":[ @@ -623,69 +88,75 @@ func doPodResizeResourceQuotaTests() { ]}}` ginkgo.By("Creating a ResourceQuota") - _, rqErr := f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Create(context.TODO(), &resourceQuota, metav1.CreateOptions{}) + _, rqErr := f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Create(ctx, &resourceQuota, metav1.CreateOptions{}) framework.ExpectNoError(rqErr, "failed to create resource quota") tStamp := strconv.Itoa(time.Now().Nanosecond()) - initDefaultResizePolicy(containers) - initDefaultResizePolicy(expected) - testPod1 := makeTestPod(f.Namespace.Name, "testpod1", tStamp, containers) - testPod2 := makeTestPod(f.Namespace.Name, "testpod2", tStamp, containers) + e2epod.InitDefaultResizePolicy(containers) + e2epod.InitDefaultResizePolicy(expected) + testPod1 := e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, containers) + testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1) + testPod2 := e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, containers) + testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2) ginkgo.By("creating pods") newPod1 := podClient.CreateSync(ctx, testPod1) newPod2 := podClient.CreateSync(ctx, testPod2) ginkgo.By("verifying initial pod resources, allocations, and policy are as expected") - verifyPodResources(newPod1, containers) + e2epod.VerifyPodResources(newPod1, containers) ginkgo.By("patching pod for resize within resource quota") - patchedPod, pErr := f.ClientSet.CoreV1().Pods(newPod1.Namespace).Patch(context.TODO(), newPod1.Name, + patchedPod, pErr := f.ClientSet.CoreV1().Pods(newPod1.Namespace).Patch(ctx, newPod1.Name, types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}) framework.ExpectNoError(pErr, "failed to patch pod for resize") ginkgo.By("verifying pod patched for resize within resource quota") - verifyPodResources(patchedPod, expected) - verifyPodAllocations(patchedPod, containers, true) + e2epod.VerifyPodResources(patchedPod, expected) + gomega.Eventually(ctx, e2epod.VerifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + WithArguments(patchedPod, containers). + Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") ginkgo.By("waiting for resize to be actuated") - resizedPod := waitForPodResizeActuation(f.ClientSet, podClient, newPod1, patchedPod, expected, containers, false) - if !isInPlaceResizeSupportedByRuntime(f.ClientSet, newPod1.Spec.NodeName) { - ginkgo.By("verifying pod container's cgroup values after resize") - if !framework.NodeOSDistroIs("windows") { - verifyPodContainersCgroupValues(resizedPod, expected, true) - } - } + resizedPod := e2epod.WaitForPodResizeActuation(ctx, f, podClient, newPod1, patchedPod, expected, containers, false) + ginkgo.By("verifying pod container's cgroup values after resize") + framework.ExpectNoError(e2epod.VerifyPodContainersCgroupValues(ctx, f, resizedPod, expected)) ginkgo.By("verifying pod resources after resize") - verifyPodResources(resizedPod, expected) + e2epod.VerifyPodResources(resizedPod, expected) ginkgo.By("verifying pod allocations after resize") - verifyPodAllocations(resizedPod, expected, true) + gomega.Eventually(ctx, e2epod.VerifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + WithArguments(resizedPod, expected). + Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") ginkgo.By("patching pod for resize with memory exceeding resource quota") - _, pErrExceedMemory := f.ClientSet.CoreV1().Pods(resizedPod.Namespace).Patch(context.TODO(), + _, pErrExceedMemory := f.ClientSet.CoreV1().Pods(resizedPod.Namespace).Patch(ctx, resizedPod.Name, types.StrategicMergePatchType, []byte(patchStringExceedMemory), metav1.PatchOptions{}) gomega.Expect(pErrExceedMemory).To(gomega.HaveOccurred(), "exceeded quota: %s, requested: memory=350Mi, used: memory=700Mi, limited: memory=800Mi", resourceQuota.Name) ginkgo.By("verifying pod patched for resize exceeding memory resource quota remains unchanged") - patchedPodExceedMemory, pErrEx2 := podClient.Get(context.TODO(), resizedPod.Name, metav1.GetOptions{}) + patchedPodExceedMemory, pErrEx2 := podClient.Get(ctx, resizedPod.Name, metav1.GetOptions{}) framework.ExpectNoError(pErrEx2, "failed to get pod post exceed memory resize") - verifyPodResources(patchedPodExceedMemory, expected) - verifyPodAllocations(patchedPodExceedMemory, expected, true) + e2epod.VerifyPodResources(patchedPodExceedMemory, expected) + gomega.Eventually(ctx, e2epod.VerifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + WithArguments(patchedPodExceedMemory, expected). + Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") ginkgo.By(fmt.Sprintf("patching pod %s for resize with CPU exceeding resource quota", resizedPod.Name)) - _, pErrExceedCPU := f.ClientSet.CoreV1().Pods(resizedPod.Namespace).Patch(context.TODO(), + _, pErrExceedCPU := f.ClientSet.CoreV1().Pods(resizedPod.Namespace).Patch(ctx, resizedPod.Name, types.StrategicMergePatchType, []byte(patchStringExceedCPU), metav1.PatchOptions{}) gomega.Expect(pErrExceedCPU).To(gomega.HaveOccurred(), "exceeded quota: %s, requested: cpu=200m, used: cpu=700m, limited: cpu=800m", resourceQuota.Name) ginkgo.By("verifying pod patched for resize exceeding CPU resource quota remains unchanged") - patchedPodExceedCPU, pErrEx1 := podClient.Get(context.TODO(), resizedPod.Name, metav1.GetOptions{}) + patchedPodExceedCPU, pErrEx1 := podClient.Get(ctx, resizedPod.Name, metav1.GetOptions{}) framework.ExpectNoError(pErrEx1, "failed to get pod post exceed CPU resize") - verifyPodResources(patchedPodExceedCPU, expected) - verifyPodAllocations(patchedPodExceedCPU, expected, true) + e2epod.VerifyPodResources(patchedPodExceedCPU, expected) + gomega.Eventually(ctx, e2epod.VerifyPodAllocations, timeouts.PodStartShort, timeouts.Poll). + WithArguments(patchedPodExceedCPU, expected). + Should(gomega.BeNil(), "failed to verify Pod allocations for patchedPod") ginkgo.By("deleting pods") delErr1 := e2epod.DeletePodWithWait(ctx, f.ClientSet, newPod1) @@ -703,6 +174,13 @@ func doPodResizeSchedulerTests() { }) ginkgo.It("pod-resize-scheduler-tests", func(ctx context.Context) { + ginkgo.By("check if in place pod vertical scaling is supported", func() { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + if !e2epod.IsInPlacePodVerticalScalingSupportedByRuntime(node) || framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + }) nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) framework.ExpectNoError(err, "failed to get running nodes") gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty()) @@ -719,7 +197,7 @@ func doPodResizeSchedulerTests() { // Exclude pods that are in the Succeeded or Failed states selector := fmt.Sprintf("spec.nodeName=%s,status.phase!=%v,status.phase!=%v", n.Name, v1.PodSucceeded, v1.PodFailed) listOptions := metav1.ListOptions{FieldSelector: selector} - podList, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceAll).List(context.TODO(), listOptions) + podList, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceAll).List(ctx, listOptions) framework.ExpectNoError(err, "failed to get running pods") framework.Logf("Found %d pods on node '%s'", len(podList.Items), n.Name) @@ -750,16 +228,16 @@ func doPodResizeSchedulerTests() { framework.Logf("TEST1: testPod2 initial CPU request is '%dm'", testPod2CPUQuantity.MilliValue()) framework.Logf("TEST1: testPod2 resized CPU request is '%dm'", testPod2CPUQuantityResized.MilliValue()) - c1 := []TestContainerInfo{ + c1 := []e2epod.ResizableContainerInfo{ { Name: "c1", - Resources: &ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()}, + Resources: &e2epod.ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()}, }, } - c2 := []TestContainerInfo{ + c2 := []e2epod.ResizableContainerInfo{ { Name: "c2", - Resources: &ContainerResources{CPUReq: testPod2CPUQuantity.String(), CPULim: testPod2CPUQuantity.String()}, + Resources: &e2epod.ContainerResources{CPUReq: testPod2CPUQuantity.String(), CPULim: testPod2CPUQuantity.String()}, }, } patchTestpod2ToFitNode := fmt.Sprintf(`{ @@ -774,10 +252,12 @@ func doPodResizeSchedulerTests() { }`, testPod2CPUQuantityResized.MilliValue(), testPod2CPUQuantityResized.MilliValue()) tStamp := strconv.Itoa(time.Now().Nanosecond()) - initDefaultResizePolicy(c1) - initDefaultResizePolicy(c2) - testPod1 := makeTestPod(f.Namespace.Name, "testpod1", tStamp, c1) - testPod2 := makeTestPod(f.Namespace.Name, "testpod2", tStamp, c2) + e2epod.InitDefaultResizePolicy(c1) + e2epod.InitDefaultResizePolicy(c2) + testPod1 := e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, c1) + testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1) + testPod2 := e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, c2) + testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2) e2epod.SetNodeAffinity(&testPod1.Spec, node.Name) e2epod.SetNodeAffinity(&testPod2.Spec, node.Name) @@ -811,10 +291,10 @@ func doPodResizeSchedulerTests() { testPod1CPUQuantityResized := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/3, resource.DecimalSI) framework.Logf("TEST2: testPod1 MilliCPUs after resize '%dm'", testPod1CPUQuantityResized.MilliValue()) - c3 := []TestContainerInfo{ + c3 := []e2epod.ResizableContainerInfo{ { Name: "c3", - Resources: &ContainerResources{CPUReq: testPod3CPUQuantity.String(), CPULim: testPod3CPUQuantity.String()}, + Resources: &e2epod.ContainerResources{CPUReq: testPod3CPUQuantity.String(), CPULim: testPod3CPUQuantity.String()}, }, } patchTestpod1ToMakeSpaceForPod3 := fmt.Sprintf(`{ @@ -829,8 +309,9 @@ func doPodResizeSchedulerTests() { }`, testPod1CPUQuantityResized.MilliValue(), testPod1CPUQuantityResized.MilliValue()) tStamp = strconv.Itoa(time.Now().Nanosecond()) - initDefaultResizePolicy(c3) - testPod3 := makeTestPod(f.Namespace.Name, "testpod3", tStamp, c3) + e2epod.InitDefaultResizePolicy(c3) + testPod3 := e2epod.MakePodWithResizableContainers(f.Namespace.Name, "testpod3", tStamp, c3) + testPod3 = e2epod.MustMixinRestrictedPodSecurity(testPod3) e2epod.SetNodeAffinity(&testPod3.Spec, node.Name) ginkgo.By(fmt.Sprintf("TEST2: Create testPod3 '%s' that cannot fit node '%s' due to insufficient CPU.", testPod3.Name, node.Name)) @@ -840,7 +321,7 @@ func doPodResizeSchedulerTests() { gomega.Expect(testPod3.Status.Phase).To(gomega.Equal(v1.PodPending)) ginkgo.By(fmt.Sprintf("TEST2: Resize pod '%s' to make enough space for pod '%s'", testPod1.Name, testPod3.Name)) - testPod1, p1Err := f.ClientSet.CoreV1().Pods(testPod1.Namespace).Patch(context.TODO(), + testPod1, p1Err := f.ClientSet.CoreV1().Pods(testPod1.Namespace).Patch(ctx, testPod1.Name, types.StrategicMergePatchType, []byte(patchTestpod1ToMakeSpaceForPod3), metav1.PatchOptions{}) framework.ExpectNoError(p1Err, "failed to patch pod for resize") From 3e6df4a871c17c1149f9b893153471c5b4a822a3 Mon Sep 17 00:00:00 2001 From: Anish Shah Date: Thu, 17 Oct 2024 12:10:22 -0700 Subject: [PATCH 4/4] test: remove container runtime check and fix other nits --- test/e2e/common/node/pod_resize.go | 55 +++++++++++----------------- test/e2e/framework/pod/resize.go | 34 ++++++------------ test/e2e/node/pod_resize.go | 57 +++++++++++++----------------- 3 files changed, 57 insertions(+), 89 deletions(-) diff --git a/test/e2e/common/node/pod_resize.go b/test/e2e/common/node/pod_resize.go index b067adf7c11..f59d87dbebe 100644 --- a/test/e2e/common/node/pod_resize.go +++ b/test/e2e/common/node/pod_resize.go @@ -23,20 +23,20 @@ import ( "strconv" "time" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" ) const ( @@ -115,13 +115,7 @@ func removeExtendedResource(clientSet clientset.Interface, nodeName, extendedRes }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) } -func doPodResizeTests() { - f := framework.NewDefaultFramework("pod-resize-test") - var podClient *e2epod.PodClient - ginkgo.BeforeEach(func() { - podClient = e2epod.NewPodClient(f) - }) - +func doPodResizeTests(f *framework.Framework) { type testCase struct { name string containers []e2epod.ResizableContainerInfo @@ -861,13 +855,7 @@ func doPodResizeTests() { for idx := range tests { tc := tests[idx] ginkgo.It(tc.name, func(ctx context.Context) { - ginkgo.By("check if in place pod vertical scaling is supported", func() { - node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) - framework.ExpectNoError(err) - if !e2epod.IsInPlacePodVerticalScalingSupportedByRuntime(node) || framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { - e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") - } - }) + podClient := e2epod.NewPodClient(f) var testPod, patchedPod *v1.Pod var pErr error @@ -945,12 +933,7 @@ func doPodResizeTests() { } } -func doPodResizeErrorTests() { - f := framework.NewDefaultFramework("pod-resize-errors") - var podClient *e2epod.PodClient - ginkgo.BeforeEach(func() { - podClient = e2epod.NewPodClient(f) - }) +func doPodResizeErrorTests(f *framework.Framework) { type testCase struct { name string @@ -985,13 +968,7 @@ func doPodResizeErrorTests() { for idx := range tests { tc := tests[idx] ginkgo.It(tc.name, func(ctx context.Context) { - ginkgo.By("check if in place pod vertical scaling is supported", func() { - node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) - framework.ExpectNoError(err) - if !e2epod.IsInPlacePodVerticalScalingSupportedByRuntime(node) || framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { - e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") - } - }) + podClient := e2epod.NewPodClient(f) var testPod, patchedPod *v1.Pod var pErr error @@ -1043,7 +1020,17 @@ func doPodResizeErrorTests() { // Above tests are performed by doSheduletTests() and doPodResizeResourceQuotaTests() // in test/e2e/node/pod_resize.go -var _ = SIGDescribe("Pod InPlace Resize Container", framework.WithSerial(), feature.InPlacePodVerticalScaling, func() { - doPodResizeTests() - doPodResizeErrorTests() +var _ = SIGDescribe("Pod InPlace Resize Container", framework.WithSerial(), feature.InPlacePodVerticalScaling, "[NodeAlphaFeature:InPlacePodVerticalScaling]", func() { + f := framework.NewDefaultFramework("pod-resize-tests") + + ginkgo.BeforeEach(func(ctx context.Context) { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + }) + + doPodResizeTests(f) + doPodResizeErrorTests(f) }) diff --git a/test/e2e/framework/pod/resize.go b/test/e2e/framework/pod/resize.go index f7b8cb54b88..8fdc5015edb 100644 --- a/test/e2e/framework/pod/resize.go +++ b/test/e2e/framework/pod/resize.go @@ -20,22 +20,20 @@ import ( "context" "encoding/json" "fmt" - "regexp" "strconv" "strings" - semver "github.com/blang/semver/v4" - "github.com/google/go-cmp/cmp" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - podutil "k8s.io/kubernetes/pkg/api/v1/pod" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/framework" imageutils "k8s.io/kubernetes/test/utils/image" + + "github.com/google/go-cmp/cmp" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" ) const ( @@ -104,21 +102,6 @@ type patchSpec struct { } `json:"spec"` } -func IsInPlacePodVerticalScalingSupportedByRuntime(node *v1.Node) bool { - re := regexp.MustCompile("containerd://(.*)") - match := re.FindStringSubmatch(node.Status.NodeInfo.ContainerRuntimeVersion) - if len(match) != 2 { - return false - } - if ver, verr := semver.ParseTolerant(match[1]); verr == nil { - if ver.Compare(semver.MustParse(MinContainerRuntimeVersion)) < 0 { - return false - } - return true - } - return false -} - func getTestResourceInfo(tcInfo ResizableContainerInfo) (v1.ResourceRequirements, v1.ResourceList, []v1.ContainerResizePolicy) { var res v1.ResourceRequirements var alloc v1.ResourceList @@ -237,6 +220,7 @@ func MakePodWithResizableContainers(ns, name, timeStamp string, tcInfo []Resizab func VerifyPodResizePolicy(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) { ginkgo.GinkgoHelper() + gomega.Expect(gotPod.Spec.Containers).To(gomega.HaveLen(len(wantCtrs)), "number of containers in pod spec should match") for i, wantCtr := range wantCtrs { gotCtr := &gotPod.Spec.Containers[i] ctr, _ := makeResizableContainer(wantCtr) @@ -247,6 +231,7 @@ func VerifyPodResizePolicy(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) { func VerifyPodResources(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) { ginkgo.GinkgoHelper() + gomega.Expect(gotPod.Spec.Containers).To(gomega.HaveLen(len(wantCtrs)), "number of containers in pod spec should match") for i, wantCtr := range wantCtrs { gotCtr := &gotPod.Spec.Containers[i] ctr, _ := makeResizableContainer(wantCtr) @@ -257,6 +242,7 @@ func VerifyPodResources(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) { func VerifyPodAllocations(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) error { ginkgo.GinkgoHelper() + gomega.Expect(gotPod.Status.ContainerStatuses).To(gomega.HaveLen(len(wantCtrs)), "number of containers in pod spec should match") for i, wantCtr := range wantCtrs { gotCtrStatus := &gotPod.Status.ContainerStatuses[i] if wantCtr.Allocations == nil { @@ -280,6 +266,7 @@ func VerifyPodAllocations(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) err func VerifyPodStatusResources(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) { ginkgo.GinkgoHelper() + gomega.Expect(gotPod.Status.ContainerStatuses).To(gomega.HaveLen(len(wantCtrs)), "number of containers in pod spec should match") for i, wantCtr := range wantCtrs { gotCtrStatus := &gotPod.Status.ContainerStatuses[i] ctr, _ := makeResizableContainer(wantCtr) @@ -288,9 +275,10 @@ func VerifyPodStatusResources(gotPod *v1.Pod, wantCtrs []ResizableContainerInfo) } } +// isPodOnCgroupv2Node checks whether the pod is running on cgroupv2 node. +// TODO: Deduplicate this function with NPD cluster e2e test: +// https://github.com/kubernetes/kubernetes/blob/2049360379bcc5d6467769cef112e6e492d3d2f0/test/e2e/node/node_problem_detector.go#L369 func isPodOnCgroupv2Node(f *framework.Framework, pod *v1.Pod) bool { - // Determine if pod is running on cgroupv2 or cgroupv1 node - //TODO(vinaykul,InPlacePodVerticalScaling): Is there a better way to determine this? cmd := "mount -t cgroup2" out, _, err := ExecCommandInContainerWithFullOutput(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", cmd) if err != nil { diff --git a/test/e2e/node/pod_resize.go b/test/e2e/node/pod_resize.go index 81c45bef154..a266bbf70f5 100644 --- a/test/e2e/node/pod_resize.go +++ b/test/e2e/node/pod_resize.go @@ -22,37 +22,26 @@ import ( "strconv" "time" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - resourceapi "k8s.io/kubernetes/pkg/api/v1/resource" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" ) -func doPodResizeResourceQuotaTests() { - f := framework.NewDefaultFramework("pod-resize-resource-quota") - var podClient *e2epod.PodClient - ginkgo.BeforeEach(func() { - podClient = e2epod.NewPodClient(f) - }) +func doPodResizeResourceQuotaTests(f *framework.Framework) { timeouts := framework.NewTimeoutContext() ginkgo.It("pod-resize-resource-quota-test", func(ctx context.Context) { - ginkgo.By("check if in place pod vertical scaling is supported", func() { - node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) - framework.ExpectNoError(err) - if !e2epod.IsInPlacePodVerticalScalingSupportedByRuntime(node) || framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { - e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") - } - }) + podClient := e2epod.NewPodClient(f) resourceQuota := v1.ResourceQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "resize-resource-quota", @@ -166,21 +155,9 @@ func doPodResizeResourceQuotaTests() { }) } -func doPodResizeSchedulerTests() { - f := framework.NewDefaultFramework("pod-resize-scheduler") - var podClient *e2epod.PodClient - ginkgo.BeforeEach(func() { - podClient = e2epod.NewPodClient(f) - }) - +func doPodResizeSchedulerTests(f *framework.Framework) { ginkgo.It("pod-resize-scheduler-tests", func(ctx context.Context) { - ginkgo.By("check if in place pod vertical scaling is supported", func() { - node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) - framework.ExpectNoError(err) - if !e2epod.IsInPlacePodVerticalScalingSupportedByRuntime(node) || framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { - e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") - } - }) + podClient := e2epod.NewPodClient(f) nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) framework.ExpectNoError(err, "failed to get running nodes") gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty()) @@ -342,9 +319,25 @@ func doPodResizeSchedulerTests() { } var _ = SIGDescribe(framework.WithSerial(), "Pod InPlace Resize Container (scheduler-focused)", feature.InPlacePodVerticalScaling, func() { - doPodResizeSchedulerTests() + f := framework.NewDefaultFramework("pod-resize-scheduler-tests") + ginkgo.BeforeEach(func(ctx context.Context) { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + }) + doPodResizeSchedulerTests(f) }) var _ = SIGDescribe("Pod InPlace Resize Container", feature.InPlacePodVerticalScaling, func() { - doPodResizeResourceQuotaTests() + f := framework.NewDefaultFramework("pod-resize-tests") + ginkgo.BeforeEach(func(ctx context.Context) { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) { + e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping") + } + }) + doPodResizeResourceQuotaTests(f) })