diff --git a/test/e2e/windows/host_process.go b/test/e2e/windows/host_process.go index 33fbb7224df..0764cfea6b8 100644 --- a/test/e2e/windows/host_process.go +++ b/test/e2e/windows/host_process.go @@ -30,6 +30,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet" e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" @@ -658,39 +660,11 @@ var _ = SIGDescribe("[Feature:WindowsHostProcessContainers] [MinimumKubeletVersi e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 1, 0, timeout) ginkgo.By("Getting container stats for pod") - nodeStats, err := e2ekubelet.GetStatsSummary(ctx, f.ClientSet, targetNode.Name) - framework.ExpectNoError(err, "Error getting node stats") - statsChecked := false - for _, podStats := range nodeStats.Pods { - if podStats.PodRef.Namespace != f.Namespace.Name { - continue - } - - // check various pod stats - if *podStats.CPU.UsageCoreNanoSeconds <= 0 { - framework.Failf("Pod %s/%s stats report cpu usage equal to %v but should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds) - } - if *podStats.Memory.WorkingSetBytes <= 0 { - framework.Failf("Pod %s/%s stats report memory usage equal to %v but should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds) - } - - for _, containerStats := range podStats.Containers { - statsChecked = true - - // check various container stats - if *containerStats.CPU.UsageCoreNanoSeconds <= 0 { - framework.Failf("Container %s stats report cpu usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.CPU.UsageCoreNanoSeconds) - } - if *containerStats.Memory.WorkingSetBytes <= 0 { - framework.Failf("Container %s stats report memory usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.Memory.WorkingSetBytes) - } - if *containerStats.Logs.UsedBytes <= 0 { - framework.Failf("Container %s stats log usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.Logs.UsedBytes) - } - } - } + wait.Poll(2*time.Second, timeout, func() (bool, error) { + return ensurePodsStatsOnNode(ctx, f.ClientSet, f.Namespace.Name, targetNode.Name, &statsChecked) + }) if !statsChecked { framework.Failf("Failed to get stats for pod %s/%s", f.Namespace.Name, podName) @@ -1003,6 +977,47 @@ type HostProcessContainersMetrics struct { StartedInitContainersErrorCount int64 } +// ensurePodsStatsOnNode returns a boolean after checking Pods statistics on a particular node +func ensurePodsStatsOnNode(ctx context.Context, c clientset.Interface, namespace, nodeName string, statsChecked *bool) (bool, error) { + nodeStats, err := e2ekubelet.GetStatsSummary(ctx, c, nodeName) + framework.ExpectNoError(err, "Error getting node stats") + + for _, podStats := range nodeStats.Pods { + if podStats.PodRef.Namespace != namespace { + continue + } + + // check various pod stats + if *podStats.CPU.UsageCoreNanoSeconds <= 0 { + framework.Logf("Pod %s/%s stats report cpu usage equal to %v but should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds) + return false, nil + } + if *podStats.Memory.WorkingSetBytes <= 0 { + framework.Logf("Pod %s/%s stats report memory usage equal to %v but should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds) + return false, nil + } + + for _, containerStats := range podStats.Containers { + *statsChecked = true + + // check various container stats + if *containerStats.CPU.UsageCoreNanoSeconds <= 0 { + framework.Logf("Container %s stats report cpu usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.CPU.UsageCoreNanoSeconds) + return false, nil + } + if *containerStats.Memory.WorkingSetBytes <= 0 { + framework.Logf("Container %s stats report memory usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.Memory.WorkingSetBytes) + return false, nil + } + if *containerStats.Logs.UsedBytes <= 0 { + framework.Logf("Container %s stats log usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.Logs.UsedBytes) + return false, nil + } + } + } + return true, nil +} + // getCurrentHostProcessMetrics returns a HostPRocessContainersMetrics object. Any metrics that do not have any // values reported will be set to 0. func getCurrentHostProcessMetrics(ctx context.Context, f *framework.Framework, nodeName string) (HostProcessContainersMetrics, error) {