diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index 5c154d68992..a3a50628840 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" clientretry "k8s.io/client-go/util/retry" + resourceapi "k8s.io/kubernetes/pkg/api/v1/resource" "k8s.io/kubernetes/test/e2e/framework" netutil "k8s.io/utils/net" ) @@ -500,6 +501,49 @@ func hasNonblockingTaint(node *v1.Node, nonblockingTaints string) bool { return false } +// GetNodeAllocatableAndAvailableQuantities with resourceName set to v1.ResourceCPU ( or set to v1.ResourceMemory ) returns +// - the first return value, upon success is the allocatable portion of a node's total "CPU" ( or "Memory" ) capacity that's available for pods to use, +// calculated by subtracting "CPU" ( or "Memory") resources reserved for node's operating system, Kubelet and other +// system daemons from the node's total "CPU" ( or "Memory" ) resource, upon failure is empty. +// - the second return value, upon success is the available portion of a node's allocatable "CPU" capacity, calculated by subtracting "CPU" +// ( or "Memory" ) resource reserved for node's pods from the node's allocatable "CPU" ( or "Memory" ) resource, upon failure is empty. +// - the third return value, upon success is nil, upon failure is the coresponding error. +func GetNodeAllocatableAndAvailableQuantities(ctx context.Context, c clientset.Interface, n *v1.Node, resourceName v1.ResourceName) (resource.Quantity, resource.Quantity, error) { + var nodeAllocatable, podAllocated, nodeAvailable resource.Quantity + switch resourceName { + case v1.ResourceCPU: + nodeAllocatable = n.Status.Allocatable.Cpu().DeepCopy() + case v1.ResourceMemory: + nodeAllocatable = n.Status.Allocatable.Memory().DeepCopy() + default: + return resource.Quantity{}, resource.Quantity{}, fmt.Errorf("unexpected resource type %q; expected either 'CPU' or 'Memory'", resourceName) + } + + if n.Status.Allocatable == nil { + return resource.Quantity{}, resource.Quantity{}, fmt.Errorf("unexpected empty value of allocatable, while attempting to get node's allocatable %s", resourceName.String()) + } + + // Exclude pods that are in the Succeeded or Failed states + selector := fmt.Sprintf("spec.nodeName=%s,status.phase!=%v,status.phase!=%v", n.Name, v1.PodSucceeded, v1.PodFailed) + listOptions := metav1.ListOptions{FieldSelector: selector} + podList, err := c.CoreV1().Pods(metav1.NamespaceAll).List(ctx, listOptions) + if err != nil { + return resource.Quantity{}, resource.Quantity{}, fmt.Errorf("error getting list of pods: %w, while attempting to get node's allocatable %s", err, resourceName.String()) + } + + for _, pod := range podList.Items { + podRequest := resourceapi.GetResourceRequestQuantity(&pod, resourceName) + podAllocated.Add(podRequest) + } + nodeAvailable = nodeAllocatable.DeepCopy() + nodeAvailable.Sub(podAllocated) + if nodeAvailable.Sign() < 0 { + return resource.Quantity{}, resource.Quantity{}, fmt.Errorf("unexpected negative value of nodeAvailable %s, while attempting to get node's allocatable and available %s", nodeAvailable.String(), resourceName.String()) + } + + return nodeAllocatable, nodeAvailable, nil +} + // GetNodeHeartbeatTime returns the timestamp of the last status update of the node. func GetNodeHeartbeatTime(node *v1.Node) metav1.Time { for _, condition := range node.Status.Conditions { @@ -533,7 +577,7 @@ func PodNodePairs(ctx context.Context, c clientset.Interface, ns string) ([]PodN func GetClusterZones(ctx context.Context, c clientset.Interface) (sets.String, error) { nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { - return nil, fmt.Errorf("Error getting nodes while attempting to list cluster zones: %w", err) + return nil, fmt.Errorf("error getting nodes while attempting to list cluster zones: %w", err) } // collect values of zone label from all nodes diff --git a/test/e2e/node/pod_resize.go b/test/e2e/node/pod_resize.go index 07bfd4cee3d..99a771f9d30 100644 --- a/test/e2e/node/pod_resize.go +++ b/test/e2e/node/pod_resize.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" helpers "k8s.io/component-helpers/resource" - resourceapi "k8s.io/kubernetes/pkg/api/v1/resource" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/common/node/framework/cgroups" "k8s.io/kubernetes/test/e2e/common/node/framework/podresize" @@ -501,9 +500,10 @@ func doPodResizeSchedulerTests(f *framework.Framework) { ginkgo.By("Find node CPU resources available for allocation!") node := nodes.Items[0] - nodeAllocatableMilliCPU, nodeAvailableMilliCPU := getNodeAllocatableAndAvailableValues(ctx, f, &node, v1.ResourceCPU) + nodeAllocatableCPU, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) + framework.ExpectNoError(err, "failed to get CPU resources available for allocation") framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", - node.Name, nodeAllocatableMilliCPU, nodeAvailableMilliCPU) + node.Name, nodeAllocatableCPU.MilliValue(), nodeAvailableCPU.MilliValue()) // // Scheduler focused pod resize E2E test case #1: @@ -511,8 +511,8 @@ func doPodResizeSchedulerTests(f *framework.Framework) { // 2. Resize pod2 down so that it fits on the node and can be scheduled. // 3. Verify that pod2 gets scheduled and comes up and running. // - testPod1CPUQuantity := resource.NewMilliQuantity(nodeAvailableMilliCPU/2, resource.DecimalSI) - testPod2CPUQuantity := resource.NewMilliQuantity(nodeAvailableMilliCPU, resource.DecimalSI) + testPod1CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue()/2, resource.DecimalSI) + testPod2CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue(), resource.DecimalSI) testPod2CPUQuantityResized := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/2, resource.DecimalSI) framework.Logf("TEST1: testPod1 initial CPU request is '%dm'", testPod1CPUQuantity.MilliValue()) framework.Logf("TEST1: testPod2 initial CPU request is '%dm'", testPod2CPUQuantity.MilliValue()) @@ -575,10 +575,11 @@ func doPodResizeSchedulerTests(f *framework.Framework) { // 2. Resize pod1 down so that pod3 gets room to be scheduled. // 3. Verify that pod3 is scheduled and running. // - nodeAllocatableMilliCPU2, nodeAvailableMilliCPU2 := getNodeAllocatableAndAvailableValues(ctx, f, &node, v1.ResourceCPU) + nodeAllocatableCPU2, nodeAvailableCPU2, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) + framework.ExpectNoError(err, "failed to get CPU resources available for allocation") framework.Logf("TEST2: Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", - node.Name, nodeAllocatableMilliCPU2, nodeAvailableMilliCPU2) - testPod3CPUQuantity := resource.NewMilliQuantity(nodeAvailableMilliCPU2+testPod1CPUQuantity.MilliValue()/4, resource.DecimalSI) + node.Name, nodeAllocatableCPU2.MilliValue(), nodeAvailableCPU2.MilliValue()) + testPod3CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU2.MilliValue()+testPod1CPUQuantity.MilliValue()/4, resource.DecimalSI) testPod1CPUQuantityResized := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/3, resource.DecimalSI) framework.Logf("TEST2: testPod1 MilliCPUs after resize '%dm'", testPod1CPUQuantityResized.MilliValue()) @@ -639,7 +640,7 @@ func doPodResizeSchedulerTests(f *framework.Framework) { } }`, testPod1CPUQuantity.MilliValue(), testPod1CPUQuantity.MilliValue()) - testPod1CPUExceedingAllocatable := resource.NewMilliQuantity(nodeAllocatableMilliCPU*2, resource.DecimalSI) + testPod1CPUExceedingAllocatable := resource.NewMilliQuantity(nodeAllocatableCPU.MilliValue()*2, resource.DecimalSI) patchTestpod1ExceedNodeAllocatable := fmt.Sprintf(`{ "spec": { "containers": [ @@ -702,11 +703,12 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { ginkgo.By("Find node CPU resources available for allocation!") node := nodes.Items[0] - nodeAllocatableMilliCPU, nodeAvailableMilliCPU := getNodeAllocatableAndAvailableValues(ctx, f, &node, v1.ResourceCPU) + nodeAllocatableCPU, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) + framework.ExpectNoError(err, "failed to get CPU resources available for allocation") framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", - node.Name, nodeAllocatableMilliCPU, nodeAvailableMilliCPU) + node.Name, nodeAllocatableCPU.MilliValue(), nodeAvailableCPU.MilliValue()) - testPod1CPUQuantity := resource.NewMilliQuantity(nodeAvailableMilliCPU/2, resource.DecimalSI) + testPod1CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue()/2, resource.DecimalSI) testPod2CPUQuantity := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/2, resource.DecimalSI) framework.Logf("testPod1 initial CPU request is '%dm'", testPod1CPUQuantity.MilliValue()) framework.Logf("testPod2 initial CPU request is '%dm'", testPod2CPUQuantity.MilliValue()) @@ -741,12 +743,13 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { gomega.Expect(testPod2.Status.Phase).To(gomega.Equal(v1.PodRunning)) gomega.Expect(testPod2.Generation).To(gomega.BeEquivalentTo(1)) - nodeAllocatableMilliCPU2, nodeAvailableMilliCPU2 := getNodeAllocatableAndAvailableValues(ctx, f, &node, v1.ResourceCPU) + nodeAllocatableCPU2, nodeAvailableCPU2, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) + framework.ExpectNoError(err, "failed to get CPU resources available for allocation") framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", - node.Name, nodeAllocatableMilliCPU2, nodeAvailableMilliCPU2) + node.Name, nodeAllocatableCPU2.MilliValue(), nodeAvailableCPU2.MilliValue()) - testPod3CPUQuantity := resource.NewMilliQuantity(nodeAvailableMilliCPU2/4, resource.DecimalSI) - testPod3CPUQuantityResized := resource.NewMilliQuantity(nodeAvailableMilliCPU2+testPod1CPUQuantity.MilliValue()/4, resource.DecimalSI) + testPod3CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU2.MilliValue()/4, resource.DecimalSI) + testPod3CPUQuantityResized := resource.NewMilliQuantity(nodeAvailableCPU2.MilliValue()+testPod1CPUQuantity.MilliValue()/4, resource.DecimalSI) framework.Logf("testPod3 MilliCPUs after resize '%dm'", testPod3CPUQuantityResized.MilliValue()) testPod1CPUQuantityResizedCPU := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/3, resource.DecimalSI) @@ -832,14 +835,13 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { ginkgo.By("Find node CPU and memory resources available for allocation!") node := nodes.Items[0] - nodeAllocatableMilliCPU, nodeAvailableMilliCPU := getNodeAllocatableAndAvailableValues(ctx, f, &node, v1.ResourceCPU) + nodeAllocatableCPU, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) + framework.ExpectNoError(err, "failed to get CPU resources available for allocation") framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", - node.Name, nodeAllocatableMilliCPU, nodeAvailableMilliCPU) - framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", - node.Name, nodeAllocatableMilliCPU, nodeAvailableMilliCPU) + node.Name, nodeAllocatableCPU.MilliValue(), nodeAvailableCPU.MilliValue()) - majorityCPUQuantity := resource.NewMilliQuantity(2*nodeAvailableMilliCPU/3, resource.DecimalSI) - littleCPUQuantity := resource.NewMilliQuantity(nodeAvailableMilliCPU/16, resource.DecimalSI) + majorityCPUQuantity := resource.NewMilliQuantity(2*nodeAvailableCPU.MilliValue()/3, resource.DecimalSI) + littleCPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue()/16, resource.DecimalSI) containerWithMajorityCPU := []podresize.ResizableContainerInfo{ { Name: "c", @@ -1060,16 +1062,18 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { ginkgo.By("Find node CPU and memory resources available for allocation!") node := nodes.Items[0] - nodeAllocatableMilliCPU, initNodeAvailableMilliCPU := getNodeAllocatableAndAvailableValues(ctx, f, &node, v1.ResourceCPU) + nodeAllocatableCPU, initNodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU) + framework.ExpectNoError(err, "failed to get CPU resources available for allocation") framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", - node.Name, nodeAllocatableMilliCPU, initNodeAvailableMilliCPU) + node.Name, nodeAllocatableCPU.MilliValue(), initNodeAvailableCPU.MilliValue()) - nodeAllocatableMem, initNodeAvailableMem := getNodeAllocatableAndAvailableValues(ctx, f, &node, v1.ResourceMemory) + nodeAllocatableMem, initNodeAvailableMem, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceMemory) + framework.ExpectNoError(err, "failed to get CPU resources available for allocation") framework.Logf("Node '%s': NodeAllocatable Memory = %d. Memory currently available to allocate = %d.", - node.Name, nodeAllocatableMem, initNodeAvailableMem) + node.Name, nodeAllocatableMem.Value(), initNodeAvailableMem.Value()) - initialCPUQuantity := resource.NewMilliQuantity(initNodeAvailableMilliCPU/4, resource.DecimalSI) - initialMemoryQuantity := resource.NewQuantity(initNodeAvailableMem/5, resource.DecimalSI) + initialCPUQuantity := resource.NewMilliQuantity(initNodeAvailableCPU.MilliValue()/4, resource.DecimalSI) + initialMemoryQuantity := resource.NewQuantity(initNodeAvailableMem.Value()/5, resource.DecimalSI) framework.Logf("initial CPU request is '%dm'", initialCPUQuantity.MilliValue()) framework.Logf("initial Memory request is '%d'", initialMemoryQuantity.Value()) @@ -1104,13 +1108,13 @@ func doPodResizeRetryDeferredTests(f *framework.Framework) { ginkgo.By(fmt.Sprintf("Create pods that fits the node '%s'", node.Name)) podClient.CreateBatch(ctx, testPods) - testPod2CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableMilliCPU/3, resource.DecimalSI) - testPod2MemoryQuantityResized := resource.NewQuantity(initNodeAvailableMem/24, resource.DecimalSI) + testPod2CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableCPU.MilliValue()/3, resource.DecimalSI) + testPod2MemoryQuantityResized := resource.NewQuantity(initNodeAvailableMem.Value()/24, resource.DecimalSI) - testPod3CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableMilliCPU/24, resource.DecimalSI) - testPod3MemoryQuantityResized := resource.NewQuantity(2*initNodeAvailableMem/3, resource.DecimalSI) + testPod3CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableCPU.MilliValue()/24, resource.DecimalSI) + testPod3MemoryQuantityResized := resource.NewQuantity(2*initNodeAvailableMem.Value()/3, resource.DecimalSI) - testPod4CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableMilliCPU/2+initNodeAvailableMilliCPU/8, resource.DecimalSI) + testPod4CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableCPU.MilliValue()/2+initNodeAvailableCPU.MilliValue()/8, resource.DecimalSI) testPod4MemoryQuantityResized := initialMemoryQuantity expectedTestPod2Resized := []podresize.ResizableContainerInfo{ @@ -1238,41 +1242,6 @@ func waitForResourceQuota(ctx context.Context, c clientset.Interface, ns, quotaN })).WithTimeout(framework.PollShortTimeout).ShouldNot(gomega.BeEmpty()) } -// Calculate available resource. nodeAvailable = nodeAllocatable - sum(podAllocated). If resourceName is "CPU", the values -// returned are in MilliValues. -func getNodeAllocatableAndAvailableValues(ctx context.Context, f *framework.Framework, n *v1.Node, resourceName v1.ResourceName) (int64, int64) { - var nodeAllocatable int64 - switch resourceName { - case v1.ResourceCPU: - nodeAllocatable = n.Status.Allocatable.Cpu().MilliValue() - case v1.ResourceMemory: - nodeAllocatable = n.Status.Allocatable.Memory().Value() - default: - framework.Failf("unexpected resource type %q; expected either 'CPU' or 'Memory'", resourceName) - } - - gomega.Expect(n.Status.Allocatable).ShouldNot(gomega.BeEmpty(), "allocatable") - podAllocated := int64(0) - - // Exclude pods that are in the Succeeded or Failed states - selector := fmt.Sprintf("spec.nodeName=%s,status.phase!=%v,status.phase!=%v", n.Name, v1.PodSucceeded, v1.PodFailed) - listOptions := metav1.ListOptions{FieldSelector: selector} - podList, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceAll).List(ctx, listOptions) - - framework.ExpectNoError(err, "failed to get running pods") - framework.Logf("Found %d pods on node '%s'", len(podList.Items), n.Name) - for _, pod := range podList.Items { - podRequest := resourceapi.GetResourceRequest(&pod, resourceName) - podAllocated += podRequest - } - nodeAvailable := nodeAllocatable - podAllocated - if nodeAvailable < 0 { - framework.Failf("unexpected negative value of nodeAvailable %d", nodeAvailable) - } - - return nodeAllocatable, nodeAvailable -} - func waitForPodDeferred(ctx context.Context, f *framework.Framework, testPod *v1.Pod) { framework.ExpectNoError(e2epod.WaitForPodCondition(ctx, f.ClientSet, testPod.Namespace, testPod.Name, "display pod resize status as deferred", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) { return helpers.IsPodResizeDeferred(pod), nil