From 04f7a8600118fa26a8f897cb49f4b031a9b87149 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Mon, 28 Oct 2024 21:48:30 +0200 Subject: [PATCH] density test: adjust CPU and memory limits Adjusted limits based on recent job log: I1028 20:05:42.079182 1002 resource_usage_test.go:199] Resource usage: container cpu(cores) memory_working_set(MB) memory_rss(MB) "kubelet" 0.024 22.17 14.20 "runtime" 0.041 409.70 84.21 I1028 20:05:42.079274 1002 resource_usage_test.go:206] CPU usage of containers: container 50th% 90th% 95th% 99th% 100th% "/" N/A N/A N/A N/A N/A "runtime" 0.014 0.834 0.834 0.834 1.083 "kubelet" 0.023 0.093 0.093 0.093 0.164 Increasing 95th percentile for runtime CPU usage should also make pull-kubernetes-node-kubelet-containerd-flaky less flaky. --- test/e2e_node/density_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/e2e_node/density_test.go b/test/e2e_node/density_test.go index 92970082aa3..eb97eaef32c 100644 --- a/test/e2e_node/density_test.go +++ b/test/e2e_node/density_test.go @@ -77,18 +77,24 @@ var _ = SIGDescribe("Density", framework.WithSerial(), framework.WithSlow(), fun }) f.Context("create a batch of pods", framework.WithFlaky(), func() { - // TODO(coufon): the values are generous, set more precise limits with benchmark data - // and add more tests + // If this test case fails with am error similar to + // "container "runtime": expected 95th% usage < 0.900; got 0.941", + // it is likely that cpuLimits or memLimits need to be increased. + // Actual resource usage values can be found in the test output, e.g.: + // I1029 11:28:15.671913 1005 resource_usage_test.go:206] CPU usage of containers: + // container 50th% 90th% 95th% 99th% 100th% + // "runtime" 0.004 0.941 0.941 0.941 0.976 + // "kubelet" 0.009 0.082 0.082 0.082 0.101 dTests := []densityTest{ { podsNr: 10, interval: 0 * time.Millisecond, cpuLimits: e2ekubelet.ContainersCPUSummary{ - kubeletstatsv1alpha1.SystemContainerKubelet: {0.50: 0.30, 0.95: 0.50}, - kubeletstatsv1alpha1.SystemContainerRuntime: {0.50: 0.40, 0.95: 0.60}, + kubeletstatsv1alpha1.SystemContainerKubelet: {0.50: 0.1, 0.95: 0.20}, + kubeletstatsv1alpha1.SystemContainerRuntime: {0.50: 0.1, 0.95: 1.5}, }, memLimits: e2ekubelet.ResourceUsagePerContainer{ - kubeletstatsv1alpha1.SystemContainerKubelet: &e2ekubelet.ContainerResourceUsage{MemoryRSSInBytes: 100 * 1024 * 1024}, + kubeletstatsv1alpha1.SystemContainerKubelet: &e2ekubelet.ContainerResourceUsage{MemoryRSSInBytes: 50 * 1024 * 1024}, kubeletstatsv1alpha1.SystemContainerRuntime: &e2ekubelet.ContainerResourceUsage{MemoryRSSInBytes: 500 * 1024 * 1024}, }, // percentile limit of single pod startup latency