github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 04:33:26 +00:00
Code Issues Projects Releases Wiki Activity

Add metric for throttled requests in AWS

Browse Source
This commit is contained in:
Fabio Bertinatto 2018-04-30 15:57:04 +02:00
parent 81bf821d69
commit 5abe207eef
3 changed files with 29 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/cloudprovider/providers/aws/aws.go
View File

@ -44,7 +44,6 @@ import (
"github.com/aws/aws-sdk-go/service/kms"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/record"
@ -4329,12 +4328,3 @@ func setNodeDisk(
}
volumeMap[volumeID] = check
}
func recordAWSMetric(actionName string, timeTaken float64, err error) {
if err != nil {
awsAPIErrorMetric.With(prometheus.Labels{"request": actionName}).Inc()
} else {
awsAPIMetric.With(prometheus.Labels{"request": actionName}).Observe(timeTaken)
}
}

20
pkg/cloudprovider/providers/aws/aws_metrics.go
View File

@ -32,9 +32,29 @@ var (
Help: "AWS API errors",
},
[]string{"request"})
awsAPIThrottlesMetric = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cloudprovider_aws_api_throttled_requests_total",
Help: "AWS API throttled requests",
},
[]string{"operation_name"})
)
func recordAWSMetric(actionName string, timeTaken float64, err error) {
if err != nil {
awsAPIErrorMetric.With(prometheus.Labels{"request": actionName}).Inc()
} else {
awsAPIMetric.With(prometheus.Labels{"request": actionName}).Observe(timeTaken)
}
}
func recordAWSThrottlesMetric(operation string) {
awsAPIThrottlesMetric.With(prometheus.Labels{"operation_name": operation}).Inc()
}
func registerMetrics() {
prometheus.MustRegister(awsAPIMetric)
prometheus.MustRegister(awsAPIErrorMetric)
prometheus.MustRegister(awsAPIThrottlesMetric)
}

14
pkg/cloudprovider/providers/aws/retry_handler.go
View File

@ -69,16 +69,19 @@ func (c *CrossRequestRetryDelay) BeforeSign(r *request.Request) {
}
}
// Return a user-friendly string describing the request, for use in log messages
func describeRequest(r *request.Request) string {
service := r.ClientInfo.ServiceName
// Return the operation name, for use in log messages and metrics
func operationName(r *request.Request) string {
name := "?"
if r.Operation != nil {
name = r.Operation.Name
}
return name
}
return service + "::" + name
// Return a user-friendly string describing the request, for use in log messages
func describeRequest(r *request.Request) string {
service := r.ClientInfo.ServiceName
return service + "::" + operationName(r)
}
// Added to the AfterRetry chain; called after any error
@ -92,6 +95,7 @@ func (c *CrossRequestRetryDelay) AfterRetry(r *request.Request) {
}
if awsError.Code() == "RequestLimitExceeded" {
c.backoff.ReportError()
recordAWSThrottlesMetric(operationName(r))
glog.Warningf("Got RequestLimitExceeded error on AWS request (%s)",
describeRequest(r))
}