Add metric for throttled requests in AWS

pkg/cloudprovider/providers/aws/aws.go

@@ -44,7 +44,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/kms"
 	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/golang/glog"
-	"github.com/prometheus/client_golang/prometheus"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
 
@@ -4329,12 +4328,3 @@ func setNodeDisk(
 	}
 	volumeMap[volumeID] = check
 }
-
-func recordAWSMetric(actionName string, timeTaken float64, err error) {
-	if err != nil {
-		awsAPIErrorMetric.With(prometheus.Labels{"request": actionName}).Inc()
-	} else {
-		awsAPIMetric.With(prometheus.Labels{"request": actionName}).Observe(timeTaken)
-	}
-
-}

pkg/cloudprovider/providers/aws/aws_metrics.go

@@ -32,9 +32,29 @@ var (
 			Help: "AWS API errors",
 		},
 		[]string{"request"})
+
+	awsAPIThrottlesMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "cloudprovider_aws_api_throttled_requests_total",
+			Help: "AWS API throttled requests",
+		},
+		[]string{"operation_name"})
 )
 
+func recordAWSMetric(actionName string, timeTaken float64, err error) {
+	if err != nil {
+		awsAPIErrorMetric.With(prometheus.Labels{"request": actionName}).Inc()
+	} else {
+		awsAPIMetric.With(prometheus.Labels{"request": actionName}).Observe(timeTaken)
+	}
+}
+
+func recordAWSThrottlesMetric(operation string) {
+	awsAPIThrottlesMetric.With(prometheus.Labels{"operation_name": operation}).Inc()
+}
+
 func registerMetrics() {
 	prometheus.MustRegister(awsAPIMetric)
 	prometheus.MustRegister(awsAPIErrorMetric)
+	prometheus.MustRegister(awsAPIThrottlesMetric)
 }

pkg/cloudprovider/providers/aws/retry_handler.go

@@ -69,16 +69,19 @@ func (c *CrossRequestRetryDelay) BeforeSign(r *request.Request) {
 	}
 }
 
-// Return a user-friendly string describing the request, for use in log messages
+// Return the operation name, for use in log messages and metrics
-func describeRequest(r *request.Request) string {
+func operationName(r *request.Request) string {
-	service := r.ClientInfo.ServiceName
-
 	name := "?"
 	if r.Operation != nil {
 		name = r.Operation.Name
 	}
+	return name
+}
 
-	return service + "::" + name
+// Return a user-friendly string describing the request, for use in log messages
+func describeRequest(r *request.Request) string {
+	service := r.ClientInfo.ServiceName
+	return service + "::" + operationName(r)
 }
 
 // Added to the AfterRetry chain; called after any error
@@ -92,6 +95,7 @@ func (c *CrossRequestRetryDelay) AfterRetry(r *request.Request) {
 	}
 	if awsError.Code() == "RequestLimitExceeded" {
 		c.backoff.ReportError()
+		recordAWSThrottlesMetric(operationName(r))
 		glog.Warningf("Got RequestLimitExceeded error on AWS request (%s)",
 			describeRequest(r))
 	}