From f43f5bc821890d915d75d04d0fe009a8cca4a271 Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Wed, 24 Apr 2024 16:29:45 +0300 Subject: [PATCH] Add an e2e test Signed-off-by: Itamar Holder --- test/e2e/node/pod_resize.go | 142 ++++++++++++++++++++++++++++++++++-- 1 file changed, 134 insertions(+), 8 deletions(-) diff --git a/test/e2e/node/pod_resize.go b/test/e2e/node/pod_resize.go index 96824ae5ff7..03167d8a51f 100644 --- a/test/e2e/node/pod_resize.go +++ b/test/e2e/node/pod_resize.go @@ -31,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" clientset "k8s.io/client-go/kubernetes" podutil "k8s.io/kubernetes/pkg/api/v1/pod" resourceapi "k8s.io/kubernetes/pkg/api/v1/resource" @@ -63,14 +64,16 @@ const ( PollInterval time.Duration = 2 * time.Second PollTimeout time.Duration = 4 * time.Minute + + fakeExtendedResource = "dummy.com/dummy" ) type ContainerResources struct { - CPUReq, CPULim, MemReq, MemLim, EphStorReq, EphStorLim string + CPUReq, CPULim, MemReq, MemLim, EphStorReq, EphStorLim, ExtendedResourceReq, ExtendedResourceLim string } type ContainerAllocations struct { - CPUAlloc, MemAlloc, ephStorAlloc string + CPUAlloc, MemAlloc, ephStorAlloc, ExtendedResourceAlloc string } type TestContainerInfo struct { @@ -146,6 +149,9 @@ func getTestResourceInfo(tcInfo TestContainerInfo) (v1.ResourceRequirements, v1. if tcInfo.Resources.EphStorLim != "" { lim[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Resources.EphStorLim) } + if tcInfo.Resources.ExtendedResourceLim != "" { + lim[fakeExtendedResource] = resource.MustParse(tcInfo.Resources.ExtendedResourceLim) + } if tcInfo.Resources.CPUReq != "" { req[v1.ResourceCPU] = resource.MustParse(tcInfo.Resources.CPUReq) } @@ -155,6 +161,9 @@ func getTestResourceInfo(tcInfo TestContainerInfo) (v1.ResourceRequirements, v1. if tcInfo.Resources.EphStorReq != "" { req[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Resources.EphStorReq) } + if tcInfo.Resources.ExtendedResourceReq != "" { + req[fakeExtendedResource] = resource.MustParse(tcInfo.Resources.ExtendedResourceReq) + } res = v1.ResourceRequirements{Limits: lim, Requests: req} } if tcInfo.Allocations != nil { @@ -168,7 +177,9 @@ func getTestResourceInfo(tcInfo TestContainerInfo) (v1.ResourceRequirements, v1. if tcInfo.Allocations.ephStorAlloc != "" { alloc[v1.ResourceEphemeralStorage] = resource.MustParse(tcInfo.Allocations.ephStorAlloc) } - + if tcInfo.Allocations.ExtendedResourceAlloc != "" { + alloc[fakeExtendedResource] = resource.MustParse(tcInfo.Allocations.ExtendedResourceAlloc) + } } if tcInfo.CPUPolicy != nil { cpuPol := v1.ContainerResizePolicy{ResourceName: v1.ResourceCPU, RestartPolicy: *tcInfo.CPUPolicy} @@ -318,7 +329,8 @@ func verifyPodAllocations(pod *v1.Pod, tcInfo []TestContainerInfo, flagError boo cStatus := cStatusMap[ci.Name] if ci.Allocations == nil { if ci.Resources != nil { - alloc := &ContainerAllocations{CPUAlloc: ci.Resources.CPUReq, MemAlloc: ci.Resources.MemReq} + alloc := &ContainerAllocations{CPUAlloc: ci.Resources.CPUReq, MemAlloc: ci.Resources.MemReq, + ExtendedResourceAlloc: ci.Resources.ExtendedResourceReq} ci.Allocations = alloc defer func() { ci.Allocations = nil @@ -571,18 +583,92 @@ func genPatchString(containers []TestContainerInfo) (string, error) { return string(patchBytes), nil } +func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error { + oldData, err := json.Marshal(old) + if err != nil { + return err + } + + newData, err := json.Marshal(new) + if err != nil { + return err + } + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{}) + if err != nil { + return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err) + } + _, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") + return err +} + +func addExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string, extendedResourceQuantity resource.Quantity) { + extendedResource := v1.ResourceName(extendedResourceName) + + ginkgo.By("Adding a custom resource") + OriginalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + node := OriginalNode.DeepCopy() + node.Status.Capacity[extendedResource] = extendedResourceQuantity + node.Status.Allocatable[extendedResource] = extendedResourceQuantity + err = patchNode(context.Background(), clientSet, OriginalNode.DeepCopy(), node) + framework.ExpectNoError(err) + + gomega.Eventually(func() error { + node, err = clientSet.CoreV1().Nodes().Get(context.Background(), node.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + fakeResourceCapacity, exists := node.Status.Capacity[extendedResource] + if !exists { + return fmt.Errorf("node %s has no %s resource capacity", node.Name, extendedResourceName) + } + if expectedResource := resource.MustParse("123"); fakeResourceCapacity.Cmp(expectedResource) != 0 { + return fmt.Errorf("node %s has resource capacity %s, expected: %s", node.Name, fakeResourceCapacity.String(), expectedResource.String()) + } + + return nil + }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) +} + +func removeExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string) { + extendedResource := v1.ResourceName(extendedResourceName) + + ginkgo.By("Removing a custom resource") + originalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + node := originalNode.DeepCopy() + delete(node.Status.Capacity, extendedResource) + delete(node.Status.Allocatable, extendedResource) + err = patchNode(context.Background(), clientSet, originalNode.DeepCopy(), node) + framework.ExpectNoError(err) + + gomega.Eventually(func() error { + node, err = clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + if _, exists := node.Status.Capacity[extendedResource]; exists { + return fmt.Errorf("node %s has resource capacity %s which is expected to be removed", node.Name, extendedResourceName) + } + + return nil + }).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred()) +} + func doPodResizeTests() { f := framework.NewDefaultFramework("pod-resize") var podClient *e2epod.PodClient + ginkgo.BeforeEach(func() { podClient = e2epod.NewPodClient(f) }) type testCase struct { - name string - containers []TestContainerInfo - patchString string - expected []TestContainerInfo + name string + containers []TestContainerInfo + patchString string + expected []TestContainerInfo + addExtendedResource bool } noRestart := v1.NotRequired @@ -1284,6 +1370,31 @@ func doPodResizeTests() { }, }, }, + { + name: "Guaranteed QoS pod, one container - increase CPU & memory with an extended resource", + containers: []TestContainerInfo{ + { + Name: "c1", + Resources: &ContainerResources{CPUReq: "100m", CPULim: "100m", MemReq: "200Mi", MemLim: "200Mi", + ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + patchString: `{"spec":{"containers":[ + {"name":"c1", "resources":{"requests":{"cpu":"200m","memory":"400Mi"},"limits":{"cpu":"200m","memory":"400Mi"}}} + ]}}`, + expected: []TestContainerInfo{ + { + Name: "c1", + Resources: &ContainerResources{CPUReq: "200m", CPULim: "200m", MemReq: "400Mi", MemLim: "400Mi", + ExtendedResourceReq: "1", ExtendedResourceLim: "1"}, + CPUPolicy: &noRestart, + MemPolicy: &noRestart, + }, + }, + addExtendedResource: true, + }, } for idx := range tests { @@ -1291,12 +1402,23 @@ func doPodResizeTests() { ginkgo.It(tc.name, func(ctx context.Context) { var testPod, patchedPod *v1.Pod var pErr error + var nodeName string tStamp := strconv.Itoa(time.Now().Nanosecond()) initDefaultResizePolicy(tc.containers) initDefaultResizePolicy(tc.expected) testPod = makeTestPod(f.Namespace.Name, "testpod", tStamp, tc.containers) + if tc.addExtendedResource { + nodes, err := e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + + nodeName = nodes.Items[0].Name + + addExtendedResource(f.ClientSet, nodeName, fakeExtendedResource, resource.MustParse("123")) + testPod.Spec.NodeName = nodeName + } + ginkgo.By("creating pod") newPod := podClient.CreateSync(ctx, testPod) @@ -1358,6 +1480,10 @@ func doPodResizeTests() { ginkgo.By("deleting pod") err = e2epod.DeletePodWithWait(ctx, f.ClientSet, newPod) framework.ExpectNoError(err, "failed to delete pod") + + if tc.addExtendedResource { + removeExtendedResource(f.ClientSet, nodeName, fakeExtendedResource) + } }) } }