From 5abe207eefb60a07e77d1e7e8d4afa897a1cd97a Mon Sep 17 00:00:00 2001
From: Fabio Bertinatto <fbertina@redhat.com>
Date: Mon, 30 Apr 2018 15:57:04 +0200
Subject: [PATCH] Add metric for throttled requests in AWS

---
 pkg/cloudprovider/providers/aws/aws.go        | 10 ----------
 .../providers/aws/aws_metrics.go              | 20 +++++++++++++++++++
 .../providers/aws/retry_handler.go            | 14 ++++++++-----
 3 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go
index 3b6d7aff3e4..730f56c8eef 100644
--- a/pkg/cloudprovider/providers/aws/aws.go
+++ b/pkg/cloudprovider/providers/aws/aws.go
@@ -44,7 +44,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/kms"
 	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/golang/glog"
-	"github.com/prometheus/client_golang/prometheus"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
 
@@ -4329,12 +4328,3 @@ func setNodeDisk(
 	}
 	volumeMap[volumeID] = check
 }
-
-func recordAWSMetric(actionName string, timeTaken float64, err error) {
-	if err != nil {
-		awsAPIErrorMetric.With(prometheus.Labels{"request": actionName}).Inc()
-	} else {
-		awsAPIMetric.With(prometheus.Labels{"request": actionName}).Observe(timeTaken)
-	}
-
-}
diff --git a/pkg/cloudprovider/providers/aws/aws_metrics.go b/pkg/cloudprovider/providers/aws/aws_metrics.go
index 1da4b2ff8dc..aa434d92607 100644
--- a/pkg/cloudprovider/providers/aws/aws_metrics.go
+++ b/pkg/cloudprovider/providers/aws/aws_metrics.go
@@ -32,9 +32,29 @@ var (
 			Help: "AWS API errors",
 		},
 		[]string{"request"})
+
+	awsAPIThrottlesMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "cloudprovider_aws_api_throttled_requests_total",
+			Help: "AWS API throttled requests",
+		},
+		[]string{"operation_name"})
 )
 
+func recordAWSMetric(actionName string, timeTaken float64, err error) {
+	if err != nil {
+		awsAPIErrorMetric.With(prometheus.Labels{"request": actionName}).Inc()
+	} else {
+		awsAPIMetric.With(prometheus.Labels{"request": actionName}).Observe(timeTaken)
+	}
+}
+
+func recordAWSThrottlesMetric(operation string) {
+	awsAPIThrottlesMetric.With(prometheus.Labels{"operation_name": operation}).Inc()
+}
+
 func registerMetrics() {
 	prometheus.MustRegister(awsAPIMetric)
 	prometheus.MustRegister(awsAPIErrorMetric)
+	prometheus.MustRegister(awsAPIThrottlesMetric)
 }
diff --git a/pkg/cloudprovider/providers/aws/retry_handler.go b/pkg/cloudprovider/providers/aws/retry_handler.go
index 8c063f8e8a4..f403168e620 100644
--- a/pkg/cloudprovider/providers/aws/retry_handler.go
+++ b/pkg/cloudprovider/providers/aws/retry_handler.go
@@ -69,16 +69,19 @@ func (c *CrossRequestRetryDelay) BeforeSign(r *request.Request) {
 	}
 }
 
-// Return a user-friendly string describing the request, for use in log messages
-func describeRequest(r *request.Request) string {
-	service := r.ClientInfo.ServiceName
-
+// Return the operation name, for use in log messages and metrics
+func operationName(r *request.Request) string {
 	name := "?"
 	if r.Operation != nil {
 		name = r.Operation.Name
 	}
+	return name
+}
 
-	return service + "::" + name
+// Return a user-friendly string describing the request, for use in log messages
+func describeRequest(r *request.Request) string {
+	service := r.ClientInfo.ServiceName
+	return service + "::" + operationName(r)
 }
 
 // Added to the AfterRetry chain; called after any error
@@ -92,6 +95,7 @@ func (c *CrossRequestRetryDelay) AfterRetry(r *request.Request) {
 	}
 	if awsError.Code() == "RequestLimitExceeded" {
 		c.backoff.ReportError()
+		recordAWSThrottlesMetric(operationName(r))
 		glog.Warningf("Got RequestLimitExceeded error on AWS request (%s)",
 			describeRequest(r))
 	}