From d952437921049891bcf69a6aa242d99069db92fb Mon Sep 17 00:00:00 2001 From: Piotr Betkier Date: Tue, 2 May 2023 14:42:12 +0200 Subject: [PATCH] Fix flaky HPA e2e tests by not failing on context cancelled (#117669) * Fix flaky HPA e2e tests by not failing on context cancelled Consume requests are sent during test execution in a loop in a separate goroutine. Once the test completes, it is expected that a consumption request may be pending. Cancelling the request during cleanup should not cause test failures. Tests started being flaky since #112923 introduced passing test context that gets cancelled during cleanup. * Use PollUntilContextTimeout and restructure error ignoring logic --- .../autoscaling/autoscaling_utils.go | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index 6a0515520fd..632c61cf7a5 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -354,10 +354,7 @@ func (rc *ResourceConsumer) makeConsumeCustomMetric(ctx context.Context) { } func (rc *ResourceConsumer) sendConsumeCPURequest(ctx context.Context, millicores int) { - ctx, cancel := context.WithTimeout(ctx, framework.SingleCallTimeout) - defer cancel() - - err := wait.PollImmediateWithContext(ctx, serviceInitializationInterval, serviceInitializationTimeout, func(ctx context.Context) (bool, error) { + err := wait.PollUntilContextTimeout(ctx, serviceInitializationInterval, serviceInitializationTimeout, true, func(ctx context.Context) (bool, error) { proxyRequest, err := e2eservice.GetServicesProxyRequest(rc.clientSet, rc.clientSet.CoreV1().RESTClient().Post()) framework.ExpectNoError(err) req := proxyRequest.Namespace(rc.nsName). @@ -375,15 +372,18 @@ func (rc *ResourceConsumer) sendConsumeCPURequest(ctx context.Context, millicore return true, nil }) + // Test has already finished (ctx got canceled), so don't fail on err from PollUntilContextTimeout + // which is a side-effect to context cancelling from the cleanup task. + if ctx.Err() != nil { + return + } + framework.ExpectNoError(err) } // sendConsumeMemRequest sends POST request for memory consumption func (rc *ResourceConsumer) sendConsumeMemRequest(ctx context.Context, megabytes int) { - ctx, cancel := context.WithTimeout(ctx, framework.SingleCallTimeout) - defer cancel() - - err := wait.PollImmediateWithContext(ctx, serviceInitializationInterval, serviceInitializationTimeout, func(ctx context.Context) (bool, error) { + err := wait.PollUntilContextTimeout(ctx, serviceInitializationInterval, serviceInitializationTimeout, true, func(ctx context.Context) (bool, error) { proxyRequest, err := e2eservice.GetServicesProxyRequest(rc.clientSet, rc.clientSet.CoreV1().RESTClient().Post()) framework.ExpectNoError(err) req := proxyRequest.Namespace(rc.nsName). @@ -401,15 +401,18 @@ func (rc *ResourceConsumer) sendConsumeMemRequest(ctx context.Context, megabytes return true, nil }) + // Test has already finished (ctx got canceled), so don't fail on err from PollUntilContextTimeout + // which is a side-effect to context cancelling from the cleanup task. + if ctx.Err() != nil { + return + } + framework.ExpectNoError(err) } // sendConsumeCustomMetric sends POST request for custom metric consumption func (rc *ResourceConsumer) sendConsumeCustomMetric(ctx context.Context, delta int) { - ctx, cancel := context.WithTimeout(ctx, framework.SingleCallTimeout) - defer cancel() - - err := wait.PollImmediateWithContext(ctx, serviceInitializationInterval, serviceInitializationTimeout, func(ctx context.Context) (bool, error) { + err := wait.PollUntilContextTimeout(ctx, serviceInitializationInterval, serviceInitializationTimeout, true, func(ctx context.Context) (bool, error) { proxyRequest, err := e2eservice.GetServicesProxyRequest(rc.clientSet, rc.clientSet.CoreV1().RESTClient().Post()) framework.ExpectNoError(err) req := proxyRequest.Namespace(rc.nsName). @@ -427,6 +430,13 @@ func (rc *ResourceConsumer) sendConsumeCustomMetric(ctx context.Context, delta i } return true, nil }) + + // Test has already finished (ctx got canceled), so don't fail on err from PollUntilContextTimeout + // which is a side-effect to context cancelling from the cleanup task. + if ctx.Err() != nil { + return + } + framework.ExpectNoError(err) }