From 9f344f23fbe1c994e26cb0298cb51bb4804f8f71 Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Mon, 28 Aug 2023 15:17:41 +0300 Subject: [PATCH 1/9] Add NodeSwap as a node feature in nodefeature.go Also, Remove wrong documentation about tempSetCurrentKubeletConfig() returning bool Signed-off-by: Itamar Holder --- test/e2e/nodefeature/nodefeature.go | 4 ++++ test/e2e_node/util.go | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/e2e/nodefeature/nodefeature.go b/test/e2e/nodefeature/nodefeature.go index 1a1f425a91a..d69e1a8b84d 100644 --- a/test/e2e/nodefeature/nodefeature.go +++ b/test/e2e/nodefeature/nodefeature.go @@ -97,6 +97,10 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) RuntimeHandler = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("RuntimeHandler")) + // Added to test Swap Feature + // This label should be used when testing KEP-2400 (Node Swap Support) + Swap = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeSwap")) + // TODO: document the feature (owning SIG, when to use this feature for a test) SidecarContainers = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("SidecarContainers")) diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index 716795f1db4..3d6fe0cf1e3 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -188,7 +188,6 @@ func cleanupPods(f *framework.Framework) { // Must be called within a Context. Allows the function to modify the KubeletConfiguration during the BeforeEach of the context. // The change is reverted in the AfterEach of the context. -// Returns true on success. func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration)) { var oldCfg *kubeletconfig.KubeletConfiguration From eb5d6476559b05ea48a334ce4b80ae01d0fa9f77 Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Wed, 28 Feb 2024 10:39:29 +0200 Subject: [PATCH 2/9] Move current test under its own NodeConformance context Signed-off-by: Itamar Holder --- test/e2e_node/swap_test.go | 59 ++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/test/e2e_node/swap_test.go b/test/e2e_node/swap_test.go index ea3c4139a20..df28411e0ed 100644 --- a/test/e2e_node/swap_test.go +++ b/test/e2e_node/swap_test.go @@ -19,6 +19,7 @@ package e2enode import ( "context" "fmt" + "k8s.io/kubernetes/test/e2e/nodefeature" "path/filepath" "strconv" @@ -44,40 +45,42 @@ const ( cgroupV1MemLimitFile = "/memory/memory.limit_in_bytes" ) -var _ = SIGDescribe("Swap", framework.WithNodeConformance(), "[LinuxOnly]", func() { - f := framework.NewDefaultFramework("swap-test") +var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { + f := framework.NewDefaultFramework("swap-qos") f.NamespacePodSecurityLevel = admissionapi.LevelBaseline - ginkgo.DescribeTable("with configuration", func(qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) { - ginkgo.By(fmt.Sprintf("Creating a pod of QOS class %s. memoryRequestEqualLimit: %t", qosClass, memoryRequestEqualLimit)) - pod := getSwapTestPod(f, qosClass, memoryRequestEqualLimit) - pod = runPodAndWaitUntilScheduled(f, pod) + f.Context(framework.WithNodeConformance(), func() { + ginkgo.DescribeTable("with configuration", func(qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) { + ginkgo.By(fmt.Sprintf("Creating a pod of QOS class %s. memoryRequestEqualLimit: %t", qosClass, memoryRequestEqualLimit)) + pod := getSwapTestPod(f, qosClass, memoryRequestEqualLimit) + pod = runPodAndWaitUntilScheduled(f, pod) - isCgroupV2 := isPodCgroupV2(f, pod) - isLimitedSwap := isLimitedSwap(f, pod) - isNoSwap := isNoSwap(f, pod) + isCgroupV2 := isPodCgroupV2(f, pod) + isLimitedSwap := isLimitedSwap(f, pod) + isNoSwap := isNoSwap(f, pod) - if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { - ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) - expectNoSwap(f, pod, isCgroupV2) - return - } + if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { + ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) + expectNoSwap(f, pod, isCgroupV2) + return + } - if !isLimitedSwap { - ginkgo.By("expecting no swap") - expectNoSwap(f, pod, isCgroupV2) - return - } + if !isLimitedSwap { + ginkgo.By("expecting no swap") + expectNoSwap(f, pod, isCgroupV2) + return + } - ginkgo.By("expecting limited swap") - expectedSwapLimit := calcSwapForBurstablePod(f, pod) - expectLimitedSwap(f, pod, expectedSwapLimit) - }, - ginkgo.Entry("QOS Best-effort", v1.PodQOSBestEffort, false), - ginkgo.Entry("QOS Burstable", v1.PodQOSBurstable, false), - ginkgo.Entry("QOS Burstable with memory request equals to limit", v1.PodQOSBurstable, true), - ginkgo.Entry("QOS Guaranteed", v1.PodQOSGuaranteed, false), - ) + ginkgo.By("expecting limited swap") + expectedSwapLimit := calcSwapForBurstablePod(f, pod) + expectLimitedSwap(f, pod, expectedSwapLimit) + }, + ginkgo.Entry("QOS Best-effort", v1.PodQOSBestEffort, false), + ginkgo.Entry("QOS Burstable", v1.PodQOSBurstable, false), + ginkgo.Entry("QOS Burstable with memory request equals to limit", v1.PodQOSBurstable, true), + ginkgo.Entry("QOS Guaranteed", v1.PodQOSGuaranteed, false), + ) + }) }) // Note that memoryRequestEqualLimit is effective only when qosClass is PodQOSBestEffort. From 959d01cbbfe22599f61487102ca8d6ca6c15eec8 Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Wed, 28 Feb 2024 11:23:41 +0200 Subject: [PATCH 3/9] Remove cgroup v1 support for swap tests Signed-off-by: Itamar Holder --- test/e2e_node/swap_test.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/test/e2e_node/swap_test.go b/test/e2e_node/swap_test.go index df28411e0ed..4472ef83e23 100644 --- a/test/e2e_node/swap_test.go +++ b/test/e2e_node/swap_test.go @@ -40,9 +40,7 @@ import ( const ( cgroupBasePath = "/sys/fs/cgroup/" - cgroupV1SwapLimitFile = "/memory/memory.memsw.limit_in_bytes" cgroupV2SwapLimitFile = "memory.swap.max" - cgroupV1MemLimitFile = "/memory/memory.limit_in_bytes" ) var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { @@ -61,13 +59,13 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) - expectNoSwap(f, pod, isCgroupV2) + expectNoSwap(f, pod) return } if !isLimitedSwap { ginkgo.By("expecting no swap") - expectNoSwap(f, pod, isCgroupV2) + expectNoSwap(f, pod) return } @@ -168,16 +166,9 @@ func isPodCgroupV2(f *framework.Framework, pod *v1.Pod) bool { return output == "true" } -func expectNoSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) { - if isCgroupV2 { - swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile) - gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("0"), "max swap allowed should be zero") - } else { - swapPlusMemLimit := readCgroupFile(f, pod, cgroupV1SwapLimitFile) - memLimit := readCgroupFile(f, pod, cgroupV1MemLimitFile) - gomega.ExpectWithOffset(1, swapPlusMemLimit).ToNot(gomega.BeEmpty()) - gomega.ExpectWithOffset(1, swapPlusMemLimit).To(gomega.Equal(memLimit)) - } +func expectNoSwap(f *framework.Framework, pod *v1.Pod) { + swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile) + gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("0"), "max swap allowed should be zero") } // supports v2 only as v1 shouldn't support LimitedSwap From 13403e836a24468fe8bb085570b8b90793bf373b Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Wed, 28 Feb 2024 11:57:14 +0200 Subject: [PATCH 4/9] Fix swap feature gate check by introduting IsFeatureGateEnabled() Signed-off-by: Itamar Holder --- test/e2e/framework/skipper/skipper.go | 13 +++++++++++-- test/e2e_node/swap_test.go | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/test/e2e/framework/skipper/skipper.go b/test/e2e/framework/skipper/skipper.go index 4a509b90a57..7d3b3d6b3de 100644 --- a/test/e2e/framework/skipper/skipper.go +++ b/test/e2e/framework/skipper/skipper.go @@ -56,8 +56,7 @@ func SkipUnlessAtLeast(value int, minValue int, message string) { var featureGate featuregate.FeatureGate // InitFeatureGates must be called in test suites that have a --feature-gates parameter. -// If not called, SkipUnlessFeatureGateEnabled and SkipIfFeatureGateEnabled will -// record a test failure. +// If not called, SkipUnlessFeatureGateEnabled will record a test failure. func InitFeatureGates(defaults featuregate.FeatureGate, overrides map[string]bool) error { clone := defaults.DeepCopy() if err := clone.SetFromMap(overrides); err != nil { @@ -67,6 +66,16 @@ func InitFeatureGates(defaults featuregate.FeatureGate, overrides map[string]boo return nil } +// IsFeatureGateEnabled can be used during e2e tests to figure out if a certain feature gate is enabled. +// This function is dependent on InitFeatureGates under the hood. Therefore, the test must be called with a +// --feature-gates parameter. +func IsFeatureGateEnabled(feature featuregate.Feature) bool { + if featureGate == nil { + framework.Failf("feature gate interface is not initialized") + } + return featureGate.Enabled(feature) +} + // SkipUnlessFeatureGateEnabled skips if the feature is disabled. // // Beware that this only works in test suites that have a --feature-gate diff --git a/test/e2e_node/swap_test.go b/test/e2e_node/swap_test.go index 4472ef83e23..a8205594ee4 100644 --- a/test/e2e_node/swap_test.go +++ b/test/e2e_node/swap_test.go @@ -19,6 +19,7 @@ package e2enode import ( "context" "fmt" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/nodefeature" "path/filepath" "strconv" @@ -29,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/test/e2e/framework" @@ -147,7 +147,7 @@ func runPodAndWaitUntilScheduled(f *framework.Framework, pod *v1.Pod) *v1.Pod { func isSwapFeatureGateEnabled() bool { ginkgo.By("figuring if NodeSwap feature gate is turned on") - return utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) + return e2eskipper.IsFeatureGateEnabled(features.NodeSwap) } func readCgroupFile(f *framework.Framework, pod *v1.Pod, filename string) string { From 2230ed7dc6ad5b46751feceb77f23be130f5babb Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Wed, 28 Feb 2024 10:32:00 +0200 Subject: [PATCH 5/9] Refactor: helper functions and quantity improvement - Add getSleepingPod() helper function - Refactor: quantity functions to return resource.quantity instead of int64 - Improve helper functions for memory capacity Signed-off-by: Itamar Holder --- test/e2e_node/swap_test.go | 65 ++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/test/e2e_node/swap_test.go b/test/e2e_node/swap_test.go index a8205594ee4..80b0a269e23 100644 --- a/test/e2e_node/swap_test.go +++ b/test/e2e_node/swap_test.go @@ -81,7 +81,7 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { }) }) -// Note that memoryRequestEqualLimit is effective only when qosClass is PodQOSBestEffort. +// Note that memoryRequestEqualLimit is effective only when qosClass is not PodQOSBestEffort. func getSwapTestPod(f *framework.Framework, qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) *v1.Pod { podMemoryAmount := resource.MustParse("128Mi") @@ -109,25 +109,28 @@ func getSwapTestPod(f *framework.Framework, qosClass v1.PodQOSClass, memoryReque resources.Requests = resources.Limits } - pod := &v1.Pod{ + pod := getSleepingPod(f.Namespace.Name) + + return pod +} + +func getSleepingPod(namespace string) *v1.Pod { + return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-swap-" + rand.String(5), - Namespace: f.Namespace.Name, + Name: "sleeping-test-pod-swap-" + rand.String(5), + Namespace: namespace, }, Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyAlways, Containers: []v1.Container{ { - Name: "busybox-container", - Image: busyboxImage, - Command: []string{"sleep", "600"}, - Resources: resources, + Name: "busybox-container", + Image: busyboxImage, + Command: []string{"sleep", "600"}, }, }, }, } - - return pod } func runPodAndWaitUntilScheduled(f *framework.Framework, pod *v1.Pod) *v1.Pod { @@ -191,44 +194,41 @@ func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit in ) } -func getSwapCapacity(f *framework.Framework, pod *v1.Pod) int64 { +func getSwapCapacity(f *framework.Framework, pod *v1.Pod) *resource.Quantity { output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", "free -b | grep Swap | xargs | cut -d\" \" -f2") - swapCapacity, err := strconv.Atoi(output) + swapCapacityBytes, err := strconv.Atoi(output) framework.ExpectNoError(err, "cannot convert swap size to int") - ginkgo.By(fmt.Sprintf("providing swap capacity: %d", swapCapacity)) + ginkgo.By(fmt.Sprintf("providing swap capacity: %d", swapCapacityBytes)) - return int64(swapCapacity) + return resource.NewQuantity(int64(swapCapacityBytes), resource.BinarySI) } -func getMemoryCapacity(f *framework.Framework, pod *v1.Pod) int64 { - nodes, err := f.ClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - framework.ExpectNoError(err, "failed listing nodes") +func getMemoryCapacity(f *framework.Framework, nodeName string) (memCapacity, usedMemory *resource.Quantity) { + node, err := f.ClientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err, fmt.Sprintf("failed getting node %s", nodeName)) - for _, node := range nodes.Items { - if node.Name != pod.Spec.NodeName { - continue - } + nodeOrigCapacity := node.Status.Capacity[v1.ResourceMemory] + memCapacity = cloneQuantity(nodeOrigCapacity) + usedMemory = cloneQuantity(nodeOrigCapacity) - memCapacity := node.Status.Capacity[v1.ResourceMemory] - return memCapacity.Value() - } - - framework.ExpectNoError(fmt.Errorf("node %s wasn't found", pod.Spec.NodeName)) - return 0 + usedMemory.Sub(node.Status.Allocatable[v1.ResourceMemory]) + return } func calcSwapForBurstablePod(f *framework.Framework, pod *v1.Pod) int64 { - nodeMemoryCapacity := getMemoryCapacity(f, pod) - nodeSwapCapacity := getSwapCapacity(f, pod) + gomega.Expect(pod.Spec.NodeName).ToNot(gomega.BeEmpty(), "pod node name is empty") + + nodeMemoryCapacityQuantity, _ := getMemoryCapacity(f, pod.Spec.NodeName) + nodeMemoryCapacity := nodeMemoryCapacityQuantity.Value() + nodeSwapCapacity := getSwapCapacity(f, pod).Value() containerMemoryRequest := pod.Spec.Containers[0].Resources.Requests.Memory().Value() containerMemoryProportion := float64(containerMemoryRequest) / float64(nodeMemoryCapacity) swapAllocation := containerMemoryProportion * float64(nodeSwapCapacity) ginkgo.By(fmt.Sprintf("Calculating swap for burstable pods: nodeMemoryCapacity: %d, nodeSwapCapacity: %d, containerMemoryRequest: %d, swapAllocation: %d", nodeMemoryCapacity, nodeSwapCapacity, containerMemoryRequest, int64(swapAllocation))) - return int64(swapAllocation) } @@ -245,3 +245,8 @@ func isNoSwap(f *framework.Framework, pod *v1.Pod) bool { return kubeletCfg.MemorySwap.SwapBehavior == types.NoSwap || kubeletCfg.MemorySwap.SwapBehavior == "" } + +func cloneQuantity(resource resource.Quantity) *resource.Quantity { + clone := resource.DeepCopy() + return &clone +} From bdeb80a846144c8f8d7d4f636f70fcf7a4f8a511 Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Mon, 4 Mar 2024 15:23:09 +0200 Subject: [PATCH 6/9] Add serial tests Signed-off-by: Itamar Holder --- test/e2e_node/swap_test.go | 369 ++++++++++++++++++++++++++++++++++--- 1 file changed, 345 insertions(+), 24 deletions(-) diff --git a/test/e2e_node/swap_test.go b/test/e2e_node/swap_test.go index 80b0a269e23..d289cc2ce10 100644 --- a/test/e2e_node/swap_test.go +++ b/test/e2e_node/swap_test.go @@ -19,10 +19,17 @@ package e2enode import ( "context" "fmt" + "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" + "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/kubelet/cm" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/nodefeature" + "math/big" + "os/exec" "path/filepath" "strconv" + "strings" + "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -39,8 +46,15 @@ import ( ) const ( - cgroupBasePath = "/sys/fs/cgroup/" - cgroupV2SwapLimitFile = "memory.swap.max" + cgroupBasePath = "/sys/fs/cgroup/" + cgroupV2SwapLimitFile = "memory.swap.max" + cgroupV2swapCurrentUsageFile = "memory.swap.current" + cgroupV2MemoryCurrentUsageFile = "memory.current" +) + +var ( + noRequests *resource.Quantity = nil + noLimits *resource.Quantity = nil ) var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { @@ -59,19 +73,19 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) - expectNoSwap(f, pod) + expectNoSwap(pod) return } if !isLimitedSwap { ginkgo.By("expecting no swap") - expectNoSwap(f, pod) + expectNoSwap(pod) return } ginkgo.By("expecting limited swap") expectedSwapLimit := calcSwapForBurstablePod(f, pod) - expectLimitedSwap(f, pod, expectedSwapLimit) + expectLimitedSwap(pod, expectedSwapLimit) }, ginkgo.Entry("QOS Best-effort", v1.PodQOSBestEffort, false), ginkgo.Entry("QOS Burstable", v1.PodQOSBurstable, false), @@ -79,6 +93,168 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { ginkgo.Entry("QOS Guaranteed", v1.PodQOSGuaranteed, false), ) }) + + f.Context(framework.WithSerial(), func() { + // These tests assume the following, and will fail otherwise: + // - The node is provisioned with swap + // - The node is configured with cgroup v2 + // - The swap feature gate is enabled + // - The node has no more than 15GB of memory + f.Context("with swap stress", func() { + var ( + nodeName string + nodeTotalMemory *resource.Quantity + nodeUsedMemory *resource.Quantity + swapCapacity *resource.Quantity + stressMemAllocSize *resource.Quantity + podClient *e2epod.PodClient + ) + + ginkgo.BeforeEach(func() { + podClient = e2epod.NewPodClient(f) + + sleepingPod := getSleepingPod(f.Namespace.Name) + sleepingPod = runPodAndWaitUntilScheduled(f, sleepingPod) + gomega.Expect(isPodCgroupV2(f, sleepingPod)).To(gomega.BeTrue(), "node uses cgroup v1") + + nodeName = sleepingPod.Spec.NodeName + gomega.Expect(nodeName).ToNot(gomega.BeEmpty(), "node name is empty") + + nodeTotalMemory, nodeUsedMemory = getMemoryCapacity(f, nodeName) + gomega.Expect(nodeTotalMemory.IsZero()).To(gomega.BeFalse(), "node memory capacity is zero") + gomega.Expect(nodeUsedMemory.IsZero()).To(gomega.BeFalse(), "node used memory is zero") + + swapCapacity = getSwapCapacity(f, sleepingPod) + gomega.Expect(swapCapacity.IsZero()).To(gomega.BeFalse(), "node swap capacity is zero") + + err := podClient.Delete(context.Background(), sleepingPod.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + stressMemAllocSize = multiplyQuantity(divideQuantity(nodeTotalMemory, 1000), 4) + + ginkgo.By(fmt.Sprintf("Setting node values. nodeName: %s, nodeTotalMemory: %s, nodeUsedMemory: %s, swapCapacity: %s, stressMemAllocSize: %s", + nodeName, nodeTotalMemory.String(), nodeUsedMemory.String(), swapCapacity.String(), stressMemAllocSize.String())) + }) + + ginkgo.Context("LimitedSwap", func() { + tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *config.KubeletConfiguration) { + msg := "swap behavior is already set to LimitedSwap" + + if swapBehavior := initialConfig.MemorySwap.SwapBehavior; swapBehavior != types.LimitedSwap { + initialConfig.MemorySwap.SwapBehavior = types.LimitedSwap + msg = "setting swap behavior to LimitedSwap" + } + + ginkgo.By(msg) + }) + + getRequestBySwapLimit := func(swapPercentage int64) *resource.Quantity { + gomega.ExpectWithOffset(1, swapPercentage).To(gomega.And( + gomega.BeNumerically(">=", 1), + gomega.BeNumerically("<=", 100), + ), "percentage has to be between 1 and 100") + + // = *(/). + // if x is the percentage, and == (x/100)*, then: + // = (x/100)**(/) = (x/100)*. + return multiplyQuantity(divideQuantity(nodeTotalMemory, 100), swapPercentage) + } + + getStressPod := func(stressSize *resource.Quantity) *v1.Pod { + pod := getStressPod(f, stressSize, stressMemAllocSize) + pod.Spec.NodeName = nodeName + + return pod + } + + ginkgo.It("should be able over-commit the node memory", func() { + stressSize := cloneQuantity(nodeTotalMemory) + + stressPod := getStressPod(stressSize) + // Request will use a lot more swap memory than needed, since we don't test swap limits in this test + memRequest := getRequestBySwapLimit(30) + setPodMmoryResources(stressPod, memRequest, noLimits) + gomega.Expect(qos.GetPodQOS(stressPod)).To(gomega.Equal(v1.PodQOSBurstable)) + + ginkgo.By(fmt.Sprintf("creating a stress pod with stress size %s and request of %s", stressSize.String(), memRequest.String())) + stressPod = runPodAndWaitUntilScheduled(f, stressPod) + + ginkgo.By("Expecting the swap usage to be non-zero") + var swapUsage *resource.Quantity + gomega.Eventually(func() error { + stressPod = getUpdatedPod(f, stressPod) + gomega.Expect(stressPod.Status.Phase).To(gomega.Equal(v1.PodRunning), "pod should be running") + + var err error + swapUsage, err = getSwapUsage(stressPod) + if err != nil { + return err + } + + if swapUsage.IsZero() { + return fmt.Errorf("swap usage is zero") + } + + return nil + }, 5*time.Minute, 1*time.Second).Should(gomega.Succeed(), "swap usage is above zero: %s", swapUsage.String()) + + // Better to delete the stress pod ASAP to avoid node failures + err := podClient.Delete(context.Background(), stressPod.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + }) + + ginkgo.It("should be able to use more memory than memory limits", func() { + stressSize := divideQuantity(nodeTotalMemory, 5) + ginkgo.By("Creating a stress pod with stress size: " + stressSize.String()) + stressPod := getStressPod(stressSize) + + memoryLimit := cloneQuantity(stressSize) + memoryLimit.Sub(resource.MustParse("50Mi")) + memoryRequest := divideQuantity(memoryLimit, 2) + ginkgo.By("Adding memory request of " + memoryRequest.String() + " and memory limit of " + memoryLimit.String()) + setPodMmoryResources(stressPod, memoryRequest, memoryLimit) + gomega.Expect(qos.GetPodQOS(stressPod)).To(gomega.Equal(v1.PodQOSBurstable)) + + var swapUsage, memoryUsage *resource.Quantity + // This is sanity check to ensure that swap usage is not caused by a system-level pressure, but + // due to a container-level (cgroup-level) pressure that's caused because of the memory limits. + minExpectedMemoryUsage := multiplyQuantity(divideQuantity(memoryLimit, 4), 3) + + stressPod = runPodAndWaitUntilScheduled(f, stressPod) + + ginkgo.By("Expecting the pod exceed limits and avoid an OOM kill since it would use swap") + gomega.Eventually(func() error { + stressPod = getUpdatedPod(f, stressPod) + gomega.Expect(stressPod.Status.Phase).To(gomega.Equal(v1.PodRunning), "pod should be running") + + var err error + swapUsage, err = getSwapUsage(stressPod) + if err != nil { + return err + } + + memoryUsage, err = getMemoryUsage(stressPod) + if err != nil { + return err + } + + if memoryUsage.Cmp(*minExpectedMemoryUsage) == -1 { + return fmt.Errorf("memory usage (%s) is smaller than minimum expected memory usage (%s)", memoryUsage.String(), minExpectedMemoryUsage.String()) + } + if swapUsage.IsZero() { + return fmt.Errorf("swap usage is zero") + } + + return nil + }, 5*time.Minute, 1*time.Second).Should(gomega.Succeed()) + + // Better to delete the stress pod ASAP to avoid node failures + err := podClient.Delete(context.Background(), stressPod.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + }) + }) + }) + }) }) // Note that memoryRequestEqualLimit is effective only when qosClass is not PodQOSBestEffort. @@ -133,14 +309,41 @@ func getSleepingPod(namespace string) *v1.Pod { } } +func getStressPod(f *framework.Framework, stressSize, memAllocSize *resource.Quantity) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "stress-pod-" + rand.String(5), + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "stress-container", + Image: "registry.k8s.io/stress:v1", + ImagePullPolicy: v1.PullAlways, + Args: []string{"-mem-alloc-size", memAllocSize.String(), "-mem-alloc-sleep", "500ms", "-mem-total", strconv.Itoa(int(stressSize.Value()))}, + }, + }, + }, + } +} + +func getUpdatedPod(f *framework.Framework, pod *v1.Pod) *v1.Pod { + podClient := e2epod.NewPodClient(f) + pod, err := podClient.Get(context.Background(), pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + return pod +} + func runPodAndWaitUntilScheduled(f *framework.Framework, pod *v1.Pod) *v1.Pod { ginkgo.By("running swap test pod") podClient := e2epod.NewPodClient(f) pod = podClient.CreateSync(context.Background(), pod) - pod, err := podClient.Get(context.Background(), pod.Name, metav1.GetOptions{}) + pod = getUpdatedPod(f, pod) - framework.ExpectNoError(err) isReady, err := testutils.PodRunningReady(pod) framework.ExpectNoError(err) gomega.ExpectWithOffset(1, isReady).To(gomega.BeTrue(), "pod should be ready") @@ -153,15 +356,6 @@ func isSwapFeatureGateEnabled() bool { return e2eskipper.IsFeatureGateEnabled(features.NodeSwap) } -func readCgroupFile(f *framework.Framework, pod *v1.Pod, filename string) string { - filePath := filepath.Join(cgroupBasePath, filename) - - ginkgo.By("reading cgroup file " + filePath) - output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", "cat "+filePath) - - return output -} - func isPodCgroupV2(f *framework.Framework, pod *v1.Pod) bool { ginkgo.By("figuring is test pod runs cgroup v2") output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", `if test -f "/sys/fs/cgroup/cgroup.controllers"; then echo "true"; else echo "false"; fi`) @@ -169,14 +363,20 @@ func isPodCgroupV2(f *framework.Framework, pod *v1.Pod) bool { return output == "true" } -func expectNoSwap(f *framework.Framework, pod *v1.Pod) { - swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile) - gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("0"), "max swap allowed should be zero") +func expectNoSwap(pod *v1.Pod) { + ginkgo.By("expecting no swap") + const offest = 1 + + swapLimit, err := readCgroupFile(pod, cgroupV2SwapLimitFile) + gomega.ExpectWithOffset(offest, err).ToNot(gomega.HaveOccurred()) + gomega.ExpectWithOffset(offest, swapLimit).To(gomega.Equal("0"), "max swap allowed should be zero") } // supports v2 only as v1 shouldn't support LimitedSwap -func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) { - swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile) +func expectLimitedSwap(pod *v1.Pod, expectedSwapLimit int64) { + ginkgo.By("expecting limited swap") + swapLimitStr, err := readCgroupFile(pod, cgroupV2SwapLimitFile) + framework.ExpectNoError(err) swapLimit, err := strconv.Atoi(swapLimitStr) framework.ExpectNoError(err, "cannot convert swap limit to int") @@ -210,8 +410,8 @@ func getMemoryCapacity(f *framework.Framework, nodeName string) (memCapacity, us framework.ExpectNoError(err, fmt.Sprintf("failed getting node %s", nodeName)) nodeOrigCapacity := node.Status.Capacity[v1.ResourceMemory] - memCapacity = cloneQuantity(nodeOrigCapacity) - usedMemory = cloneQuantity(nodeOrigCapacity) + memCapacity = cloneQuantity(&nodeOrigCapacity) + usedMemory = cloneQuantity(&nodeOrigCapacity) usedMemory.Sub(node.Status.Allocatable[v1.ResourceMemory]) return @@ -246,7 +446,128 @@ func isNoSwap(f *framework.Framework, pod *v1.Pod) bool { return kubeletCfg.MemorySwap.SwapBehavior == types.NoSwap || kubeletCfg.MemorySwap.SwapBehavior == "" } -func cloneQuantity(resource resource.Quantity) *resource.Quantity { +func cloneQuantity(resource *resource.Quantity) *resource.Quantity { clone := resource.DeepCopy() return &clone } + +func divideQuantity(quantity *resource.Quantity, divideBy int64) *resource.Quantity { + dividedBigInt := new(big.Int).Div( + quantity.AsDec().UnscaledBig(), + big.NewInt(divideBy), + ) + + return resource.NewQuantity(dividedBigInt.Int64(), quantity.Format) +} + +func multiplyQuantity(quantity *resource.Quantity, multiplier int64) *resource.Quantity { + product := new(big.Int).Mul( + quantity.AsDec().UnscaledBig(), + big.NewInt(multiplier), + ) + + return resource.NewQuantity(product.Int64(), quantity.Format) +} + +func multiplyQuantities(quantity1, quantity2 *resource.Quantity) *resource.Quantity { + product := new(big.Int).Mul(quantity1.AsDec().UnscaledBig(), quantity2.AsDec().UnscaledBig()) + + return resource.NewQuantity(product.Int64(), quantity1.Format) +} + +func getPodCgroupPath(pod *v1.Pod) string { + podQos := qos.GetPodQOS(pod) + cgroupQosComponent := "" + + switch podQos { + case v1.PodQOSBestEffort: + cgroupQosComponent = bestEffortCgroup + case v1.PodQOSBurstable: + cgroupQosComponent = burstableCgroup + } + + var rootCgroupName cm.CgroupName + if cgroupQosComponent != "" { + rootCgroupName = cm.NewCgroupName(cm.RootCgroupName, defaultNodeAllocatableCgroup, cgroupQosComponent) + } else { + rootCgroupName = cm.NewCgroupName(cm.RootCgroupName, defaultNodeAllocatableCgroup) + } + + cgroupsToVerify := "pod" + string(pod.UID) + cgroupName := cm.NewCgroupName(rootCgroupName, cgroupsToVerify) + cgroupFsPath := toCgroupFsName(cgroupName) + + return filepath.Join(cgroupBasePath, cgroupFsPath) +} + +func readCgroupFile(pod *v1.Pod, cgroupFile string) (string, error) { + cgroupPath := getPodCgroupPath(pod) + cgroupFilePath := filepath.Join(cgroupPath, cgroupFile) + + ginkgo.By("Reading cgroup file: " + cgroupFilePath) + cmd := "cat " + cgroupFilePath + outputBytes, err := exec.Command("sudo", "sh", "-c", cmd).CombinedOutput() + if err != nil { + return "", fmt.Errorf("error running cmd %s: %w", cmd, err) + } + + outputStr := strings.TrimSpace(string(outputBytes)) + ginkgo.By("cgroup found value: " + outputStr) + + return outputStr, nil +} + +func parseBytesStrToQuantity(bytesStr string) (*resource.Quantity, error) { + bytesInt, err := strconv.ParseInt(bytesStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing swap usage %s to int: %w", bytesStr, err) + } + + return resource.NewQuantity(bytesInt, resource.BinarySI), nil +} + +func getSwapUsage(pod *v1.Pod) (*resource.Quantity, error) { + outputStr, err := readCgroupFile(pod, cgroupV2swapCurrentUsageFile) + if err != nil { + return nil, err + } + + ginkgo.By("swap usage found: " + outputStr + " bytes") + + return parseBytesStrToQuantity(outputStr) +} + +func getMemoryUsage(pod *v1.Pod) (*resource.Quantity, error) { + outputStr, err := readCgroupFile(pod, cgroupV2MemoryCurrentUsageFile) + if err != nil { + return nil, err + } + + ginkgo.By("memory usage found: " + outputStr + " bytes") + + return parseBytesStrToQuantity(outputStr) +} + +// Sets memory request or limit can be null, then it's dismissed. +// Sets the same value for all containers. +func setPodMmoryResources(pod *v1.Pod, memoryRequest, memoryLimit *resource.Quantity) { + for i := range pod.Spec.Containers { + resources := &pod.Spec.Containers[i].Resources + + if memoryRequest != nil { + if resources.Requests == nil { + resources.Requests = make(map[v1.ResourceName]resource.Quantity) + } + + resources.Requests[v1.ResourceMemory] = *memoryRequest + } + + if memoryLimit != nil { + if resources.Limits == nil { + resources.Limits = make(map[v1.ResourceName]resource.Quantity) + } + + resources.Limits[v1.ResourceMemory] = *memoryLimit + } + } +} From b17050927cb6fbc351d3ea45eee613af9a97c8f4 Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Wed, 6 Mar 2024 13:28:03 +0200 Subject: [PATCH 7/9] Update node conformance to use NoSwap Signed-off-by: Itamar Holder --- test/e2e_node/swap_test.go | 46 ++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/test/e2e_node/swap_test.go b/test/e2e_node/swap_test.go index d289cc2ce10..e4583ed4cc8 100644 --- a/test/e2e_node/swap_test.go +++ b/test/e2e_node/swap_test.go @@ -61,31 +61,34 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { f := framework.NewDefaultFramework("swap-qos") f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + ginkgo.BeforeEach(func() { + gomega.Expect(isSwapFeatureGateEnabled()).To(gomega.BeTrueBecause("NodeSwap feature should be on")) + }) + f.Context(framework.WithNodeConformance(), func() { + ginkgo.DescribeTable("with configuration", func(qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) { ginkgo.By(fmt.Sprintf("Creating a pod of QOS class %s. memoryRequestEqualLimit: %t", qosClass, memoryRequestEqualLimit)) pod := getSwapTestPod(f, qosClass, memoryRequestEqualLimit) pod = runPodAndWaitUntilScheduled(f, pod) - isCgroupV2 := isPodCgroupV2(f, pod) - isLimitedSwap := isLimitedSwap(f, pod) - isNoSwap := isNoSwap(f, pod) + gomega.Expect(isPodCgroupV2(f, pod)).To(gomega.BeTrueBecause("cgroup v2 is required for swap")) - if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { - ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) + switch swapBehavior := getSwapBehavior(); swapBehavior { + case types.LimitedSwap: + if qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit { + expectNoSwap(pod) + } else { + expectedSwapLimit := calcSwapForBurstablePod(f, pod) + expectLimitedSwap(pod, expectedSwapLimit) + } + + case types.NoSwap, "": expectNoSwap(pod) - return - } - if !isLimitedSwap { - ginkgo.By("expecting no swap") - expectNoSwap(pod) - return + default: + gomega.Expect(swapBehavior).To(gomega.Or(gomega.Equal(types.LimitedSwap), gomega.Equal(types.NoSwap)), "unknown swap behavior") } - - ginkgo.By("expecting limited swap") - expectedSwapLimit := calcSwapForBurstablePod(f, pod) - expectLimitedSwap(pod, expectedSwapLimit) }, ginkgo.Entry("QOS Best-effort", v1.PodQOSBestEffort, false), ginkgo.Entry("QOS Burstable", v1.PodQOSBurstable, false), @@ -432,18 +435,13 @@ func calcSwapForBurstablePod(f *framework.Framework, pod *v1.Pod) int64 { return int64(swapAllocation) } -func isLimitedSwap(f *framework.Framework, pod *v1.Pod) bool { +func getSwapBehavior() string { kubeletCfg, err := getCurrentKubeletConfig(context.Background()) framework.ExpectNoError(err, "cannot get kubelet config") - return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap -} - -func isNoSwap(f *framework.Framework, pod *v1.Pod) bool { - kubeletCfg, err := getCurrentKubeletConfig(context.Background()) - framework.ExpectNoError(err, "cannot get kubelet config") - - return kubeletCfg.MemorySwap.SwapBehavior == types.NoSwap || kubeletCfg.MemorySwap.SwapBehavior == "" + swapBehavior := kubeletCfg.MemorySwap.SwapBehavior + ginkgo.By("Figuring out swap behavior: " + swapBehavior) + return swapBehavior } func cloneQuantity(resource *resource.Quantity) *resource.Quantity { From ab5f84e8edb18b8827b75ad2dccb1e1989990351 Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Tue, 12 Mar 2024 13:54:04 +0200 Subject: [PATCH 8/9] Refactor: Better pod image, read cgroup file from container - Improve cgroup file read: execute from container instead of host - Clean unused variables/functions - BeTrue/BeFalse -> BeTrueBecause/BeFalseBecause - Use agnhost instread of stress image - Improve description and fix typo Signed-off-by: Itamar Holder --- test/e2e_node/swap_test.go | 102 +++++++++++-------------------------- 1 file changed, 31 insertions(+), 71 deletions(-) diff --git a/test/e2e_node/swap_test.go b/test/e2e_node/swap_test.go index e4583ed4cc8..623e132f5f6 100644 --- a/test/e2e_node/swap_test.go +++ b/test/e2e_node/swap_test.go @@ -21,11 +21,10 @@ import ( "fmt" "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" "k8s.io/kubernetes/pkg/kubelet/apis/config" - "k8s.io/kubernetes/pkg/kubelet/cm" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/nodefeature" + imageutils "k8s.io/kubernetes/test/utils/image" "math/big" - "os/exec" "path/filepath" "strconv" "strings" @@ -53,8 +52,7 @@ const ( ) var ( - noRequests *resource.Quantity = nil - noLimits *resource.Quantity = nil + noLimits *resource.Quantity = nil ) var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { @@ -77,14 +75,14 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { switch swapBehavior := getSwapBehavior(); swapBehavior { case types.LimitedSwap: if qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit { - expectNoSwap(pod) + expectNoSwap(f, pod) } else { expectedSwapLimit := calcSwapForBurstablePod(f, pod) - expectLimitedSwap(pod, expectedSwapLimit) + expectLimitedSwap(f, pod, expectedSwapLimit) } case types.NoSwap, "": - expectNoSwap(pod) + expectNoSwap(f, pod) default: gomega.Expect(swapBehavior).To(gomega.Or(gomega.Equal(types.LimitedSwap), gomega.Equal(types.NoSwap)), "unknown swap behavior") @@ -118,17 +116,17 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { sleepingPod := getSleepingPod(f.Namespace.Name) sleepingPod = runPodAndWaitUntilScheduled(f, sleepingPod) - gomega.Expect(isPodCgroupV2(f, sleepingPod)).To(gomega.BeTrue(), "node uses cgroup v1") + gomega.Expect(isPodCgroupV2(f, sleepingPod)).To(gomega.BeTrueBecause("node uses cgroup v1")) nodeName = sleepingPod.Spec.NodeName gomega.Expect(nodeName).ToNot(gomega.BeEmpty(), "node name is empty") nodeTotalMemory, nodeUsedMemory = getMemoryCapacity(f, nodeName) - gomega.Expect(nodeTotalMemory.IsZero()).To(gomega.BeFalse(), "node memory capacity is zero") - gomega.Expect(nodeUsedMemory.IsZero()).To(gomega.BeFalse(), "node used memory is zero") + gomega.Expect(nodeTotalMemory.IsZero()).To(gomega.BeFalseBecause("node memory capacity is zero")) + gomega.Expect(nodeUsedMemory.IsZero()).To(gomega.BeFalseBecause("node used memory is zero")) swapCapacity = getSwapCapacity(f, sleepingPod) - gomega.Expect(swapCapacity.IsZero()).To(gomega.BeFalse(), "node swap capacity is zero") + gomega.Expect(swapCapacity.IsZero()).To(gomega.BeFalseBecause("node swap capacity is zero")) err := podClient.Delete(context.Background(), sleepingPod.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) @@ -170,13 +168,13 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { return pod } - ginkgo.It("should be able over-commit the node memory", func() { + ginkgo.It("should be able to use more than the node memory capacity", func() { stressSize := cloneQuantity(nodeTotalMemory) stressPod := getStressPod(stressSize) // Request will use a lot more swap memory than needed, since we don't test swap limits in this test memRequest := getRequestBySwapLimit(30) - setPodMmoryResources(stressPod, memRequest, noLimits) + setPodMemoryResources(stressPod, memRequest, noLimits) gomega.Expect(qos.GetPodQOS(stressPod)).To(gomega.Equal(v1.PodQOSBurstable)) ginkgo.By(fmt.Sprintf("creating a stress pod with stress size %s and request of %s", stressSize.String(), memRequest.String())) @@ -189,7 +187,7 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { gomega.Expect(stressPod.Status.Phase).To(gomega.Equal(v1.PodRunning), "pod should be running") var err error - swapUsage, err = getSwapUsage(stressPod) + swapUsage, err = getSwapUsage(f, stressPod) if err != nil { return err } @@ -215,7 +213,7 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { memoryLimit.Sub(resource.MustParse("50Mi")) memoryRequest := divideQuantity(memoryLimit, 2) ginkgo.By("Adding memory request of " + memoryRequest.String() + " and memory limit of " + memoryLimit.String()) - setPodMmoryResources(stressPod, memoryRequest, memoryLimit) + setPodMemoryResources(stressPod, memoryRequest, memoryLimit) gomega.Expect(qos.GetPodQOS(stressPod)).To(gomega.Equal(v1.PodQOSBurstable)) var swapUsage, memoryUsage *resource.Quantity @@ -231,12 +229,12 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { gomega.Expect(stressPod.Status.Phase).To(gomega.Equal(v1.PodRunning), "pod should be running") var err error - swapUsage, err = getSwapUsage(stressPod) + swapUsage, err = getSwapUsage(f, stressPod) if err != nil { return err } - memoryUsage, err = getMemoryUsage(stressPod) + memoryUsage, err = getMemoryUsage(f, stressPod) if err != nil { return err } @@ -323,9 +321,9 @@ func getStressPod(f *framework.Framework, stressSize, memAllocSize *resource.Qua Containers: []v1.Container{ { Name: "stress-container", - Image: "registry.k8s.io/stress:v1", + Image: imageutils.GetE2EImage(imageutils.Agnhost), ImagePullPolicy: v1.PullAlways, - Args: []string{"-mem-alloc-size", memAllocSize.String(), "-mem-alloc-sleep", "500ms", "-mem-total", strconv.Itoa(int(stressSize.Value()))}, + Args: []string{"stress", "--mem-alloc-size", memAllocSize.String(), "--mem-alloc-sleep", "1000ms", "--mem-total", strconv.Itoa(int(stressSize.Value()))}, }, }, }, @@ -349,7 +347,7 @@ func runPodAndWaitUntilScheduled(f *framework.Framework, pod *v1.Pod) *v1.Pod { isReady, err := testutils.PodRunningReady(pod) framework.ExpectNoError(err) - gomega.ExpectWithOffset(1, isReady).To(gomega.BeTrue(), "pod should be ready") + gomega.ExpectWithOffset(1, isReady).To(gomega.BeTrueBecause("pod should be ready")) return pod } @@ -366,19 +364,19 @@ func isPodCgroupV2(f *framework.Framework, pod *v1.Pod) bool { return output == "true" } -func expectNoSwap(pod *v1.Pod) { +func expectNoSwap(f *framework.Framework, pod *v1.Pod) { ginkgo.By("expecting no swap") const offest = 1 - swapLimit, err := readCgroupFile(pod, cgroupV2SwapLimitFile) + swapLimit, err := readCgroupFile(f, pod, cgroupV2SwapLimitFile) gomega.ExpectWithOffset(offest, err).ToNot(gomega.HaveOccurred()) gomega.ExpectWithOffset(offest, swapLimit).To(gomega.Equal("0"), "max swap allowed should be zero") } // supports v2 only as v1 shouldn't support LimitedSwap -func expectLimitedSwap(pod *v1.Pod, expectedSwapLimit int64) { +func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) { ginkgo.By("expecting limited swap") - swapLimitStr, err := readCgroupFile(pod, cgroupV2SwapLimitFile) + swapLimitStr, err := readCgroupFile(f, pod, cgroupV2SwapLimitFile) framework.ExpectNoError(err) swapLimit, err := strconv.Atoi(swapLimitStr) @@ -467,49 +465,11 @@ func multiplyQuantity(quantity *resource.Quantity, multiplier int64) *resource.Q return resource.NewQuantity(product.Int64(), quantity.Format) } -func multiplyQuantities(quantity1, quantity2 *resource.Quantity) *resource.Quantity { - product := new(big.Int).Mul(quantity1.AsDec().UnscaledBig(), quantity2.AsDec().UnscaledBig()) +func readCgroupFile(f *framework.Framework, pod *v1.Pod, cgroupFile string) (string, error) { + cgroupPath := filepath.Join(cgroupBasePath, cgroupFile) - return resource.NewQuantity(product.Int64(), quantity1.Format) -} - -func getPodCgroupPath(pod *v1.Pod) string { - podQos := qos.GetPodQOS(pod) - cgroupQosComponent := "" - - switch podQos { - case v1.PodQOSBestEffort: - cgroupQosComponent = bestEffortCgroup - case v1.PodQOSBurstable: - cgroupQosComponent = burstableCgroup - } - - var rootCgroupName cm.CgroupName - if cgroupQosComponent != "" { - rootCgroupName = cm.NewCgroupName(cm.RootCgroupName, defaultNodeAllocatableCgroup, cgroupQosComponent) - } else { - rootCgroupName = cm.NewCgroupName(cm.RootCgroupName, defaultNodeAllocatableCgroup) - } - - cgroupsToVerify := "pod" + string(pod.UID) - cgroupName := cm.NewCgroupName(rootCgroupName, cgroupsToVerify) - cgroupFsPath := toCgroupFsName(cgroupName) - - return filepath.Join(cgroupBasePath, cgroupFsPath) -} - -func readCgroupFile(pod *v1.Pod, cgroupFile string) (string, error) { - cgroupPath := getPodCgroupPath(pod) - cgroupFilePath := filepath.Join(cgroupPath, cgroupFile) - - ginkgo.By("Reading cgroup file: " + cgroupFilePath) - cmd := "cat " + cgroupFilePath - outputBytes, err := exec.Command("sudo", "sh", "-c", cmd).CombinedOutput() - if err != nil { - return "", fmt.Errorf("error running cmd %s: %w", cmd, err) - } - - outputStr := strings.TrimSpace(string(outputBytes)) + outputStr := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "sh", "-c", "cat "+cgroupPath) + outputStr = strings.TrimSpace(outputStr) ginkgo.By("cgroup found value: " + outputStr) return outputStr, nil @@ -524,8 +484,8 @@ func parseBytesStrToQuantity(bytesStr string) (*resource.Quantity, error) { return resource.NewQuantity(bytesInt, resource.BinarySI), nil } -func getSwapUsage(pod *v1.Pod) (*resource.Quantity, error) { - outputStr, err := readCgroupFile(pod, cgroupV2swapCurrentUsageFile) +func getSwapUsage(f *framework.Framework, pod *v1.Pod) (*resource.Quantity, error) { + outputStr, err := readCgroupFile(f, pod, cgroupV2swapCurrentUsageFile) if err != nil { return nil, err } @@ -535,8 +495,8 @@ func getSwapUsage(pod *v1.Pod) (*resource.Quantity, error) { return parseBytesStrToQuantity(outputStr) } -func getMemoryUsage(pod *v1.Pod) (*resource.Quantity, error) { - outputStr, err := readCgroupFile(pod, cgroupV2MemoryCurrentUsageFile) +func getMemoryUsage(f *framework.Framework, pod *v1.Pod) (*resource.Quantity, error) { + outputStr, err := readCgroupFile(f, pod, cgroupV2MemoryCurrentUsageFile) if err != nil { return nil, err } @@ -548,7 +508,7 @@ func getMemoryUsage(pod *v1.Pod) (*resource.Quantity, error) { // Sets memory request or limit can be null, then it's dismissed. // Sets the same value for all containers. -func setPodMmoryResources(pod *v1.Pod, memoryRequest, memoryLimit *resource.Quantity) { +func setPodMemoryResources(pod *v1.Pod, memoryRequest, memoryLimit *resource.Quantity) { for i := range pod.Spec.Containers { resources := &pod.Spec.Containers[i].Resources From e9b1a5e1855172797d54d050b3be6d89614dd7f4 Mon Sep 17 00:00:00 2001 From: Itamar Holder Date: Tue, 12 Mar 2024 13:36:54 +0200 Subject: [PATCH 9/9] Expect NoSwap on NodeConformance, test LimitedSwap only in serial tests Signed-off-by: Itamar Holder --- test/e2e_node/swap_test.go | 61 ++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/test/e2e_node/swap_test.go b/test/e2e_node/swap_test.go index 623e132f5f6..90944d13678 100644 --- a/test/e2e_node/swap_test.go +++ b/test/e2e_node/swap_test.go @@ -71,22 +71,9 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { pod = runPodAndWaitUntilScheduled(f, pod) gomega.Expect(isPodCgroupV2(f, pod)).To(gomega.BeTrueBecause("cgroup v2 is required for swap")) + gomega.Expect(getSwapBehavior()).To(gomega.Or(gomega.Equal(types.NoSwap), gomega.BeEmpty()), "NodeConformance is expected to run with NoSwap") - switch swapBehavior := getSwapBehavior(); swapBehavior { - case types.LimitedSwap: - if qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit { - expectNoSwap(f, pod) - } else { - expectedSwapLimit := calcSwapForBurstablePod(f, pod) - expectLimitedSwap(f, pod, expectedSwapLimit) - } - - case types.NoSwap, "": - expectNoSwap(f, pod) - - default: - gomega.Expect(swapBehavior).To(gomega.Or(gomega.Equal(types.LimitedSwap), gomega.Equal(types.NoSwap)), "unknown swap behavior") - } + expectNoSwap(f, pod) }, ginkgo.Entry("QOS Best-effort", v1.PodQOSBestEffort, false), ginkgo.Entry("QOS Burstable", v1.PodQOSBurstable, false), @@ -96,6 +83,39 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { }) f.Context(framework.WithSerial(), func() { + + enableLimitedSwap := func(ctx context.Context, initialConfig *config.KubeletConfiguration) { + msg := "swap behavior is already set to LimitedSwap" + + if swapBehavior := initialConfig.MemorySwap.SwapBehavior; swapBehavior != types.LimitedSwap { + initialConfig.MemorySwap.SwapBehavior = types.LimitedSwap + msg = "setting swap behavior to LimitedSwap" + } + + ginkgo.By(msg) + } + + f.Context("Basic functionality", func() { + tempSetCurrentKubeletConfig(f, enableLimitedSwap) + + ginkgo.DescribeTable("with configuration", func(qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) { + ginkgo.By(fmt.Sprintf("Creating a pod of QOS class %s. memoryRequestEqualLimit: %t", qosClass, memoryRequestEqualLimit)) + pod := getSwapTestPod(f, qosClass, memoryRequestEqualLimit) + pod = runPodAndWaitUntilScheduled(f, pod) + + gomega.Expect(isPodCgroupV2(f, pod)).To(gomega.BeTrueBecause("cgroup v2 is required for swap")) + gomega.Expect(getSwapBehavior()).To(gomega.Equal(types.LimitedSwap)) + + expectedSwapLimit := calcSwapForBurstablePod(f, pod) + expectLimitedSwap(f, pod, expectedSwapLimit) + }, + ginkgo.Entry("QOS Best-effort", v1.PodQOSBestEffort, false), + ginkgo.Entry("QOS Burstable", v1.PodQOSBurstable, false), + ginkgo.Entry("QOS Burstable with memory request equals to limit", v1.PodQOSBurstable, true), + ginkgo.Entry("QOS Guaranteed", v1.PodQOSGuaranteed, false), + ) + }) + // These tests assume the following, and will fail otherwise: // - The node is provisioned with swap // - The node is configured with cgroup v2 @@ -138,16 +158,7 @@ var _ = SIGDescribe("Swap", "[LinuxOnly]", nodefeature.Swap, func() { }) ginkgo.Context("LimitedSwap", func() { - tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *config.KubeletConfiguration) { - msg := "swap behavior is already set to LimitedSwap" - - if swapBehavior := initialConfig.MemorySwap.SwapBehavior; swapBehavior != types.LimitedSwap { - initialConfig.MemorySwap.SwapBehavior = types.LimitedSwap - msg = "setting swap behavior to LimitedSwap" - } - - ginkgo.By(msg) - }) + tempSetCurrentKubeletConfig(f, enableLimitedSwap) getRequestBySwapLimit := func(swapPercentage int64) *resource.Quantity { gomega.ExpectWithOffset(1, swapPercentage).To(gomega.And(