From f2af00f8213fe30bb41baace52292b7907c2c745 Mon Sep 17 00:00:00 2001 From: Zhou Fang Date: Fri, 12 Aug 2016 20:56:10 -0700 Subject: [PATCH] add benchmark test which does not verify limits --- test/e2e_node/density_test.go | 359 ++++++++++++++++----------- test/e2e_node/resource_collector.go | 2 +- test/e2e_node/resource_usage_test.go | 161 +++++++----- 3 files changed, 311 insertions(+), 211 deletions(-) diff --git a/test/e2e_node/density_test.go b/test/e2e_node/density_test.go index 560f45f3770..12bfe3695e2 100644 --- a/test/e2e_node/density_test.go +++ b/test/e2e_node/density_test.go @@ -49,31 +49,36 @@ const ( var _ = framework.KubeDescribe("Density [Serial] [Slow]", func() { const ( - // the data collection time of `resource collector' and the standalone cadvisor - // is not synchronizated. Therefore `resource collector' may miss data or + // The data collection time of resource collector and the standalone cadvisor + // is not synchronizated, so resource collector may miss data or // collect duplicated data - monitoringInterval = 500 * time.Millisecond - sleepBeforeCreatePods = 30 * time.Second + containerStatsPollingPeriod = 500 * time.Millisecond ) var ( ns string nodeName string + rc *ResourceCollector ) f := framework.NewDefaultFramework("density-test") - podType := "density_test_pod" BeforeEach(func() { ns = f.Namespace.Name nodeName = framework.TestContext.NodeName + // Start a standalone cadvisor pod using 'createSync', the pod is running when it returns + createCadvisorPod(f) + // Resource collector monitors fine-grain CPU/memory usage by a standalone Cadvisor with + // 1s housingkeeping interval + rc = NewResourceCollector(containerStatsPollingPeriod) }) AfterEach(func() { }) Context("create a batch of pods", func() { - // TODO(coufon): add more tests and the values are generous, set more precise limits after benchmark + // TODO(coufon): the values are generous, set more precise limits with benchmark data + // and add more tests dTests := []densityTest{ { podsNr: 10, @@ -101,102 +106,55 @@ var _ = framework.KubeDescribe("Density [Serial] [Slow]", func() { itArg := testArg It(fmt.Sprintf("latency/resource should be within limit when create %d pods with %v interval", itArg.podsNr, itArg.interval), func() { - var ( - mutex = &sync.Mutex{} - watchTimes = make(map[string]unversioned.Time, 0) - stopCh = make(chan struct{}) - ) - // create specifications of the test pods - pods := newTestPods(itArg.podsNr, ImageRegistry[pauseImage], podType) + batchLag, e2eLags := runDensityBatchTest(f, rc, itArg) - // start a standalone cadvisor pod - // it uses `createSync', so the pod is running when it returns - createCadvisorPod(f) - - // `resource collector' monitoring fine-grain CPU/memory usage by a standalone Cadvisor with - // 1s housingkeeping interval - rc := NewResourceCollector(monitoringInterval) - - // the controller watches the change of pod status - controller := newInformerWatchPod(f, mutex, watchTimes, podType) - go controller.Run(stopCh) - - // Zhou: In test we see kubelet starts while it is busy on something, as a result `syncLoop' - // does not response to pod creation immediately. Creating the first pod has a delay around 5s. - // The node status has been `ready' so `wait and check node being ready' does not help here. - // Now wait here for a grace period to have `syncLoop' be ready - time.Sleep(sleepBeforeCreatePods) - - // the density test only monitors the overhead of creating pod - // or start earliest and call `rc.Reset()' here to clear the buffer - rc.Start() - - By("Creating a batch of pods") - // it returns a map[`pod name']`creation time' as the creation timestamps - createTimes := createBatchPodWithRateControl(f, pods, itArg.interval) - - By("Waiting for all Pods to be observed by the watch...") - // checks every 10s util all pods are running. it times out ater 10min - Eventually(func() bool { - return len(watchTimes) == itArg.podsNr - }, 10*time.Minute, 10*time.Second).Should(BeTrue()) - - if len(watchTimes) < itArg.podsNr { - framework.Failf("Timeout reached waiting for all Pods to be observed by the watch.") - } - - // stop the watching controller, and the resource collector - close(stopCh) - rc.Stop() - - // data analyis - var ( - firstCreate unversioned.Time - lastRunning unversioned.Time - init = true - e2eLags = make([]framework.PodLatencyData, 0) - ) - - for name, create := range createTimes { - watch, ok := watchTimes[name] - Expect(ok).To(Equal(true)) - - e2eLags = append(e2eLags, - framework.PodLatencyData{Name: name, Latency: watch.Time.Sub(create.Time)}) - - if !init { - if firstCreate.Time.After(create.Time) { - firstCreate = create - } - if lastRunning.Time.Before(watch.Time) { - lastRunning = watch - } - } else { - init = false - firstCreate, lastRunning = create, watch - } - } - - sort.Sort(framework.LatencySlice(e2eLags)) - - // verify latency By("Verifying latency") - verifyLatency(lastRunning.Time.Sub(firstCreate.Time), e2eLags, itArg) + printAndVerifyLatency(batchLag, e2eLags, itArg, true) - // verify resource By("Verifying resource") - verifyResource(f, itArg.cpuLimits, itArg.memLimits, rc) + printAndVerifyResource(f, rc, itArg.cpuLimits, itArg.memLimits, true) + }) + } + }) + + Context("create a batch of pods [Benchmark]", func() { + dTests := []densityTest{ + { + podsNr: 10, + interval: 0 * time.Millisecond, + }, + { + podsNr: 35, + interval: 0 * time.Millisecond, + }, + { + podsNr: 105, + interval: 0 * time.Millisecond, + }, + } + + for _, testArg := range dTests { + itArg := testArg + It(fmt.Sprintf("latency/resource should be within limit when create %d pods with %v interval", + itArg.podsNr, itArg.interval), func() { + + batchLag, e2eLags := runDensityBatchTest(f, rc, itArg) + + By("Verifying latency") + printAndVerifyLatency(batchLag, e2eLags, itArg, false) + + By("Verifying resource") + printAndVerifyResource(f, rc, itArg.cpuLimits, itArg.memLimits, false) }) } }) Context("create a sequence of pods", func() { - // TODO(coufon): add more tests and the values are generous, set more precise limits after benchmark dTests := []densityTest{ { podsNr: 10, - bgPodsNr: 10, + bgPodsNr: 50, cpuLimits: framework.ContainersCPUSummary{ stats.SystemContainerKubelet: {0.50: 0.20, 0.95: 0.25}, stats.SystemContainerRuntime: {0.50: 0.40, 0.95: 0.60}, @@ -217,34 +175,46 @@ var _ = framework.KubeDescribe("Density [Serial] [Slow]", func() { itArg := testArg It(fmt.Sprintf("latency/resource should be within limit when create %d pods with %d background pods", itArg.podsNr, itArg.bgPodsNr), func() { - bgPods := newTestPods(itArg.bgPodsNr, ImageRegistry[pauseImage], "background_pod") - testPods := newTestPods(itArg.podsNr, ImageRegistry[pauseImage], podType) - createCadvisorPod(f) - rc := NewResourceCollector(monitoringInterval) + batchlag, e2eLags := runDensitySeqTest(f, rc, itArg) - By("Creating a batch of background pods") - // creatBatch is synchronized - // all pods are running when it returns - f.PodClient().CreateBatch(bgPods) - - time.Sleep(sleepBeforeCreatePods) - - // starting resource monitoring - rc.Start() - - // do a sequential creation of pod (back to back) - batchlag, e2eLags := createBatchPodSequential(f, testPods) - - rc.Stop() - - // verify latency By("Verifying latency") - verifyLatency(batchlag, e2eLags, itArg) + printAndVerifyLatency(batchlag, e2eLags, itArg, true) - // verify resource By("Verifying resource") - verifyResource(f, itArg.cpuLimits, itArg.memLimits, rc) + printAndVerifyResource(f, rc, itArg.cpuLimits, itArg.memLimits, true) + }) + } + }) + + Context("create a sequence of pods [Benchmark]", func() { + dTests := []densityTest{ + { + podsNr: 10, + bgPodsNr: 50, + }, + { + podsNr: 30, + bgPodsNr: 50, + }, + { + podsNr: 50, + bgPodsNr: 50, + }, + } + + for _, testArg := range dTests { + itArg := testArg + It(fmt.Sprintf("latency/resource should be within limit when create %d pods with %d background pods", + itArg.podsNr, itArg.bgPodsNr), func() { + + batchlag, e2eLags := runDensitySeqTest(f, rc, itArg) + + By("Verifying latency") + printAndVerifyLatency(batchlag, e2eLags, itArg, false) + + By("Verifying resource") + printAndVerifyResource(f, rc, itArg.cpuLimits, itArg.memLimits, false) }) } }) @@ -257,14 +227,116 @@ type densityTest struct { bgPodsNr int // interval between creating pod (rate control) interval time.Duration - // resource bound + // performance limits cpuLimits framework.ContainersCPUSummary memLimits framework.ResourceUsagePerContainer podStartupLimits framework.LatencyMetric podBatchStartupLimit time.Duration } -// it creates a batch of pods concurrently, uses one goroutine for each creation. +// runDensityBatchTest runs the density batch pod creation test +func runDensityBatchTest(f *framework.Framework, rc *ResourceCollector, testArg densityTest) (time.Duration, []framework.PodLatencyData) { + const ( + podType = "density_test_pod" + sleepBeforeCreatePods = 30 * time.Second + ) + var ( + mutex = &sync.Mutex{} + watchTimes = make(map[string]unversioned.Time, 0) + stopCh = make(chan struct{}) + ) + + // create test pod data structure + pods := newTestPods(testArg.podsNr, ImageRegistry[pauseImage], podType) + + // the controller watches the change of pod status + controller := newInformerWatchPod(f, mutex, watchTimes, podType) + go controller.Run(stopCh) + defer close(stopCh) + + // TODO(coufon): in the test we found kubelet starts while it is busy on something, as a result 'syncLoop' + // does not response to pod creation immediately. Creating the first pod has a delay around 5s. + // The node status has already been 'ready' so `wait and check node being ready does not help here. + // Now wait here for a grace period to let 'syncLoop' be ready + time.Sleep(sleepBeforeCreatePods) + + rc.Start() + defer rc.Stop() + + By("Creating a batch of pods") + // It returns a map['pod name']'creation time' containing the creation timestamps + createTimes := createBatchPodWithRateControl(f, pods, testArg.interval) + + By("Waiting for all Pods to be observed by the watch...") + + Eventually(func() bool { + return len(watchTimes) == testArg.podsNr + }, 10*time.Minute, 10*time.Second).Should(BeTrue()) + + if len(watchTimes) < testArg.podsNr { + framework.Failf("Timeout reached waiting for all Pods to be observed by the watch.") + } + + // Analyze results + var ( + firstCreate unversioned.Time + lastRunning unversioned.Time + init = true + e2eLags = make([]framework.PodLatencyData, 0) + ) + + for name, create := range createTimes { + watch, ok := watchTimes[name] + Expect(ok).To(Equal(true)) + + e2eLags = append(e2eLags, + framework.PodLatencyData{Name: name, Latency: watch.Time.Sub(create.Time)}) + + if !init { + if firstCreate.Time.After(create.Time) { + firstCreate = create + } + if lastRunning.Time.Before(watch.Time) { + lastRunning = watch + } + } else { + init = false + firstCreate, lastRunning = create, watch + } + } + + sort.Sort(framework.LatencySlice(e2eLags)) + batchLag := lastRunning.Time.Sub(firstCreate.Time) + + return batchLag, e2eLags +} + +// runDensitySeqTest runs the density sequential pod creation test +func runDensitySeqTest(f *framework.Framework, rc *ResourceCollector, testArg densityTest) (time.Duration, []framework.PodLatencyData) { + const ( + podType = "density_test_pod" + sleepBeforeCreatePods = 30 * time.Second + ) + bgPods := newTestPods(testArg.bgPodsNr, ImageRegistry[pauseImage], "background_pod") + testPods := newTestPods(testArg.podsNr, ImageRegistry[pauseImage], podType) + + By("Creating a batch of background pods") + + // CreatBatch is synchronized, all pods are running when it returns + f.PodClient().CreateBatch(bgPods) + + time.Sleep(sleepBeforeCreatePods) + + rc.Start() + defer rc.Stop() + + // create pods sequentially (back-to-back) + batchlag, e2eLags := createBatchPodSequential(f, testPods) + + return batchlag, e2eLags +} + +// createBatchPodWithRateControl creates a batch of pods concurrently, uses one goroutine for each creation. // between creations there is an interval for throughput control func createBatchPodWithRateControl(f *framework.Framework, pods []*api.Pod, interval time.Duration) map[string]unversioned.Time { createTimes := make(map[string]unversioned.Time) @@ -286,7 +358,7 @@ func checkPodDeleted(f *framework.Framework, podName string) error { return errors.New("Pod Not Deleted") } -// get prometheus metric `pod start latency' from kubelet +// getPodStartLatency gets prometheus metric 'pod start latency' from kubelet func getPodStartLatency(node string) (framework.KubeletLatencyMetrics, error) { latencyMetrics := framework.KubeletLatencyMetrics{} ms, err := metrics.GrabKubeletMetricsWithoutProxy(node) @@ -367,32 +439,6 @@ func newInformerWatchPod(f *framework.Framework, mutex *sync.Mutex, watchTimes m return controller } -// verifyLatency verifies that whether pod creation latency satisfies the limit. -func verifyLatency(batchLag time.Duration, e2eLags []framework.PodLatencyData, testArg densityTest) { - framework.PrintLatencies(e2eLags, "worst client e2e total latencies") - - // Zhou: do not trust `kubelet' metrics since they are not reset! - latencyMetrics, _ := getPodStartLatency(kubeletAddr) - framework.Logf("Kubelet Prometheus metrics (not reset):\n%s", framework.PrettyPrintJSON(latencyMetrics)) - - // check whether e2e pod startup time is acceptable. - podCreateLatency := framework.PodStartupLatency{Latency: framework.ExtractLatencyMetrics(e2eLags)} - framework.Logf("Pod create latency: %s", framework.PrettyPrintJSON(podCreateLatency)) - framework.ExpectNoError(verifyPodStartupLatency(testArg.podStartupLimits, podCreateLatency.Latency)) - - // check bactch pod creation latency - if testArg.podBatchStartupLimit > 0 { - Expect(batchLag <= testArg.podBatchStartupLimit).To(Equal(true), "Batch creation startup time %v exceed limit %v", - batchLag, testArg.podBatchStartupLimit) - } - - // calculate and log throughput - throughputBatch := float64(testArg.podsNr) / batchLag.Minutes() - framework.Logf("Batch creation throughput is %.1f pods/min", throughputBatch) - throughputSequential := 1.0 / e2eLags[len(e2eLags)-1].Latency.Minutes() - framework.Logf("Sequential creation throughput is %.1f pods/min", throughputSequential) -} - // createBatchPodSequential creats pods back-to-back in sequence. func createBatchPodSequential(f *framework.Framework, pods []*api.Pod) (time.Duration, []framework.PodLatencyData) { batchStartTime := unversioned.Now() @@ -401,9 +447,38 @@ func createBatchPodSequential(f *framework.Framework, pods []*api.Pod) (time.Dur create := unversioned.Now() f.PodClient().CreateSync(pod) e2eLags = append(e2eLags, - framework.PodLatencyData{Name: pod.ObjectMeta.Name, Latency: unversioned.Now().Time.Sub(create.Time)}) + framework.PodLatencyData{Name: pod.Name, Latency: unversioned.Now().Time.Sub(create.Time)}) } batchLag := unversioned.Now().Time.Sub(batchStartTime.Time) sort.Sort(framework.LatencySlice(e2eLags)) return batchLag, e2eLags } + +// printAndVerifyLatency verifies that whether pod creation latency satisfies the limit. +func printAndVerifyLatency(batchLag time.Duration, e2eLags []framework.PodLatencyData, testArg densityTest, isVerify bool) { + framework.PrintLatencies(e2eLags, "worst client e2e total latencies") + + // TODO(coufon): do not trust `kubelet' metrics since they are not reset! + latencyMetrics, _ := getPodStartLatency(kubeletAddr) + framework.Logf("Kubelet Prometheus metrics (not reset):\n%s", framework.PrettyPrintJSON(latencyMetrics)) + + // check whether e2e pod startup time is acceptable. + podCreateLatency := framework.PodStartupLatency{Latency: framework.ExtractLatencyMetrics(e2eLags)} + framework.Logf("Pod create latency: %s", framework.PrettyPrintJSON(podCreateLatency)) + + // calculate and log throughput + throughputBatch := float64(testArg.podsNr) / batchLag.Minutes() + framework.Logf("Batch creation throughput is %.1f pods/min", throughputBatch) + throughputSequential := 1.0 / e2eLags[len(e2eLags)-1].Latency.Minutes() + framework.Logf("Sequential creation throughput is %.1f pods/min", throughputSequential) + + if isVerify { + framework.ExpectNoError(verifyPodStartupLatency(testArg.podStartupLimits, podCreateLatency.Latency)) + + // check bactch pod creation latency + if testArg.podBatchStartupLimit > 0 { + Expect(batchLag <= testArg.podBatchStartupLimit).To(Equal(true), "Batch creation startup time %v exceed limit %v", + batchLag, testArg.podBatchStartupLimit) + } + } +} diff --git a/test/e2e_node/resource_collector.go b/test/e2e_node/resource_collector.go index 7699e33ac6e..9d0ab37c637 100644 --- a/test/e2e_node/resource_collector.go +++ b/test/e2e_node/resource_collector.go @@ -461,7 +461,7 @@ func (r *ResourceCollector) GetResourceSeriesWithLabels(labels map[string]string return seriesPerContainer } -// Zhou: code for getting container name of docker, copied from pkg/kubelet/cm/container_manager_linux.go +// Code for getting container name of docker, copied from pkg/kubelet/cm/container_manager_linux.go // since they are not exposed const ( kubeletProcessName = "kubelet" diff --git a/test/e2e_node/resource_usage_test.go b/test/e2e_node/resource_usage_test.go index 663a8804a23..bed8a3810be 100644 --- a/test/e2e_node/resource_usage_test.go +++ b/test/e2e_node/resource_usage_test.go @@ -35,12 +35,6 @@ var _ = framework.KubeDescribe("Resource-usage [Serial] [Slow]", func() { const ( // Interval to poll /stats/container on a node containerStatsPollingPeriod = 10 * time.Second - // The monitoring time for one test. - monitoringTime = 10 * time.Minute - // The periodic reporting period. - reportingPeriod = 5 * time.Minute - - sleepAfterCreatePods = 10 * time.Second ) var ( @@ -54,6 +48,12 @@ var _ = framework.KubeDescribe("Resource-usage [Serial] [Slow]", func() { BeforeEach(func() { ns = f.Namespace.Name om = framework.NewRuntimeOperationMonitor(f.Client) + // The test collects resource usage from a standalone Cadvisor pod. + // The Cadvsior of Kubelet has a housekeeping interval of 10s, which is too long to + // show the resource usage spikes. But changing its interval increases the overhead + // of kubelet. Hence we use a Cadvisor pod. + createCadvisorPod(f) + rc = NewResourceCollector(containerStatsPollingPeriod) }) AfterEach(func() { @@ -67,13 +67,11 @@ var _ = framework.KubeDescribe("Resource-usage [Serial] [Slow]", func() { Context("regular resource usage tracking", func() { rTests := []resourceTest{ { - podsPerNode: 10, + pods: 10, cpuLimits: framework.ContainersCPUSummary{ stats.SystemContainerKubelet: {0.50: 0.25, 0.95: 0.30}, stats.SystemContainerRuntime: {0.50: 0.30, 0.95: 0.40}, }, - // We set the memory limits generously because the distribution - // of the addon pods affect the memory usage on each node. memLimits: framework.ResourceUsagePerContainer{ stats.SystemContainerKubelet: &framework.ContainerResourceUsage{MemoryRSSInBytes: 100 * 1024 * 1024}, stats.SystemContainerRuntime: &framework.ContainerResourceUsage{MemoryRSSInBytes: 400 * 1024 * 1024}, @@ -84,70 +82,95 @@ var _ = framework.KubeDescribe("Resource-usage [Serial] [Slow]", func() { for _, testArg := range rTests { itArg := testArg - podsPerNode := itArg.podsPerNode - name := fmt.Sprintf("resource tracking for %d pods per node", podsPerNode) - - It(name, func() { - // The test collects resource usage from a standalone Cadvisor pod. - // The Cadvsior of Kubelet has a housekeeping interval of 10s, which is too long to - // show the resource usage spikes. But changing its interval increases the overhead - // of kubelet. Hence we use a Cadvisor pod. - createCadvisorPod(f) - rc = NewResourceCollector(containerStatsPollingPeriod) - rc.Start() - - By("Creating a batch of Pods") - pods := newTestPods(podsPerNode, ImageRegistry[pauseImage], "test_pod") - for _, pod := range pods { - f.PodClient().CreateSync(pod) - } - - // wait for a while to let the node be steady - time.Sleep(sleepAfterCreatePods) - - // Log once and flush the stats. - rc.LogLatest() - rc.Reset() - - By("Start monitoring resource usage") - // Periodically dump the cpu summary until the deadline is met. - // Note that without calling framework.ResourceMonitor.Reset(), the stats - // would occupy increasingly more memory. This should be fine - // for the current test duration, but we should reclaim the - // entries if we plan to monitor longer (e.g., 8 hours). - deadline := time.Now().Add(monitoringTime) - for time.Now().Before(deadline) { - timeLeft := deadline.Sub(time.Now()) - framework.Logf("Still running...%v left", timeLeft) - if timeLeft < reportingPeriod { - time.Sleep(timeLeft) - } else { - time.Sleep(reportingPeriod) - } - logPods(f.Client) - } - - rc.Stop() - - By("Reporting overall resource usage") - logPods(f.Client) - + It(fmt.Sprintf("resource tracking for %d pods per node", itArg.pods), func() { + runResourceUsageTest(f, rc, itArg) // Log and verify resource usage - verifyResource(f, itArg.cpuLimits, itArg.memLimits, rc) + printAndVerifyResource(f, rc, itArg.cpuLimits, itArg.memLimits, true) + }) + } + }) + + Context("regular resource usage tracking [Benchmark]", func() { + rTests := []resourceTest{ + { + pods: 10, + }, + { + pods: 35, + }, + { + pods: 105, + }, + } + + for _, testArg := range rTests { + itArg := testArg + + It(fmt.Sprintf("resource tracking for %d pods per node", itArg.pods), func() { + runResourceUsageTest(f, rc, itArg) + // Log and verify resource usage + printAndVerifyResource(f, rc, itArg.cpuLimits, itArg.memLimits, true) }) } }) }) type resourceTest struct { - podsPerNode int - cpuLimits framework.ContainersCPUSummary - memLimits framework.ResourceUsagePerContainer + pods int + cpuLimits framework.ContainersCPUSummary + memLimits framework.ResourceUsagePerContainer } -// verifyResource verifies whether resource usage satisfies the limit. -func verifyResource(f *framework.Framework, cpuLimits framework.ContainersCPUSummary, - memLimits framework.ResourceUsagePerContainer, rc *ResourceCollector) { +// runResourceUsageTest runs the resource usage test +func runResourceUsageTest(f *framework.Framework, rc *ResourceCollector, testArg resourceTest) { + const ( + // The monitoring time for one test + monitoringTime = 10 * time.Minute + // The periodic reporting period + reportingPeriod = 5 * time.Minute + // sleep for an interval here to measure steady data + sleepAfterCreatePods = 10 * time.Second + ) + + rc.Start() + defer rc.Stop() + + By("Creating a batch of Pods") + pods := newTestPods(testArg.pods, ImageRegistry[pauseImage], "test_pod") + f.PodClient().CreateBatch(pods) + + // wait for a while to let the node be steady + time.Sleep(sleepAfterCreatePods) + + // Log once and flush the stats. + rc.LogLatest() + rc.Reset() + + By("Start monitoring resource usage") + // Periodically dump the cpu summary until the deadline is met. + // Note that without calling framework.ResourceMonitor.Reset(), the stats + // would occupy increasingly more memory. This should be fine + // for the current test duration, but we should reclaim the + // entries if we plan to monitor longer (e.g., 8 hours). + deadline := time.Now().Add(monitoringTime) + for time.Now().Before(deadline) { + timeLeft := deadline.Sub(time.Now()) + framework.Logf("Still running...%v left", timeLeft) + if timeLeft < reportingPeriod { + time.Sleep(timeLeft) + } else { + time.Sleep(reportingPeriod) + } + logPods(f.Client) + } + + By("Reporting overall resource usage") + logPods(f.Client) +} + +// printAndVerifyResource prints the resource usage as perf data and verifies whether resource usage satisfies the limit. +func printAndVerifyResource(f *framework.Framework, rc *ResourceCollector, cpuLimits framework.ContainersCPUSummary, + memLimits framework.ResourceUsagePerContainer, isVerify bool) { nodeName := framework.TestContext.NodeName // Obtain memory PerfData @@ -158,20 +181,22 @@ func verifyResource(f *framework.Framework, cpuLimits framework.ContainersCPUSum usagePerNode := make(framework.ResourceUsagePerNode) usagePerNode[nodeName] = usagePerContainer - // Obtain cpu PerfData + // Obtain CPU PerfData cpuSummary := rc.GetCPUSummary() framework.Logf("%s", formatCPUSummary(cpuSummary)) cpuSummaryPerNode := make(framework.NodesCPUSummary) cpuSummaryPerNode[nodeName] = cpuSummary - // Log resource usage + // Print resource usage framework.PrintPerfData(framework.ResourceUsageToPerfData(usagePerNode)) framework.PrintPerfData(framework.CPUUsageToPerfData(cpuSummaryPerNode)) // Verify resource usage - verifyMemoryLimits(f.Client, memLimits, usagePerNode) - verifyCPULimits(cpuLimits, cpuSummaryPerNode) + if isVerify { + verifyMemoryLimits(f.Client, memLimits, usagePerNode) + verifyCPULimits(cpuLimits, cpuSummaryPerNode) + } } func verifyMemoryLimits(c *client.Client, expected framework.ResourceUsagePerContainer, actual framework.ResourceUsagePerNode) {