From d7735f32d0bfddbc9d2bb3d182e8ece58a3f5146 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sat, 24 Feb 2024 18:07:49 -0500 Subject: [PATCH] Skip autoscaling tests when we hit rate limits Signed-off-by: Davanum Srinivas --- .../autoscaling/custom_metrics_stackdriver_autoscaling.go | 3 +++ .../monitoring/custom_metrics_stackdriver.go | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go index f587fbe2825..1e321a3f314 100644 --- a/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go +++ b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go @@ -420,6 +420,9 @@ func (tc *CustomMetricTestCase) Run(ctx context.Context) { // Set up a cluster: create a custom metric and set up k8s-sd adapter err = monitoring.CreateDescriptors(gcmService, projectID) if err != nil { + if strings.Contains(err.Error(), "Request throttled") { + e2eskipper.Skipf("Skipping...hitting rate limits on creating and updating metrics/labels") + } framework.Failf("Failed to create metric descriptor: %v", err) } defer monitoring.CleanupDescriptors(gcmService, projectID) diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go index 4f460e24e27..15c440c2cad 100644 --- a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go +++ b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go @@ -18,6 +18,7 @@ package monitoring import ( "context" + "strings" "time" gcm "google.golang.org/api/monitoring/v3" @@ -112,6 +113,9 @@ func testCustomMetrics(ctx context.Context, f *framework.Framework, kubeClient c // Set up a cluster: create a custom metric and set up k8s-sd adapter err = CreateDescriptors(gcmService, projectID) if err != nil { + if strings.Contains(err.Error(), "Request throttled") { + e2eskipper.Skipf("Skipping...hitting rate limits on creating and updating metrics/labels") + } framework.Failf("Failed to create metric descriptor: %s", err) } ginkgo.DeferCleanup(CleanupDescriptors, gcmService, projectID) @@ -158,6 +162,9 @@ func testExternalMetrics(ctx context.Context, f *framework.Framework, kubeClient // Set up a cluster: create a custom metric and set up k8s-sd adapter err = CreateDescriptors(gcmService, projectID) if err != nil { + if strings.Contains(err.Error(), "Request throttled") { + e2eskipper.Skipf("Skipping...hitting rate limits on creating and updating metrics/labels") + } framework.Failf("Failed to create metric descriptor: %s", err) } ginkgo.DeferCleanup(CleanupDescriptors, gcmService, projectID)