From f2aa330a38a412320aa32dec6d248956de1b1649 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 24 Jan 2017 21:38:50 -0500 Subject: [PATCH] Start recording cloud provider metrics for AWS Lets start recording storage metrics for AWS. --- pkg/cloudprovider/providers/aws/BUILD | 2 + pkg/cloudprovider/providers/aws/aws.go | 55 +++++++++++++++---- .../providers/aws/aws_metrics.go | 40 ++++++++++++++ 3 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 pkg/cloudprovider/providers/aws/aws_metrics.go diff --git a/pkg/cloudprovider/providers/aws/BUILD b/pkg/cloudprovider/providers/aws/BUILD index 88e3d6b5a5e..47e16a82c45 100644 --- a/pkg/cloudprovider/providers/aws/BUILD +++ b/pkg/cloudprovider/providers/aws/BUILD @@ -14,6 +14,7 @@ go_library( "aws.go", "aws_instancegroups.go", "aws_loadbalancer.go", + "aws_metrics.go", "aws_routes.go", "aws_utils.go", "device_allocator.go", @@ -42,6 +43,7 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index 1909370a1b3..de999b9d60e 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -40,6 +40,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -571,10 +572,11 @@ func (s *awsSdkEC2) DescribeInstances(request *ec2.DescribeInstancesInput) ([]*e // Instances are paged results := []*ec2.Instance{} var nextToken *string - + requestTime := time.Now() for { response, err := s.ec2.DescribeInstances(request) if err != nil { + recordAwsMetric("describe_instance", 0, err) return nil, fmt.Errorf("error listing AWS instances: %v", err) } @@ -588,7 +590,8 @@ func (s *awsSdkEC2) DescribeInstances(request *ec2.DescribeInstancesInput) ([]*e } request.NextToken = nextToken } - + timeTaken := time.Since(requestTime).Seconds() + recordAwsMetric("describe_instance", timeTaken, nil) return results, nil } @@ -603,22 +606,31 @@ func (s *awsSdkEC2) DescribeSecurityGroups(request *ec2.DescribeSecurityGroupsIn } func (s *awsSdkEC2) AttachVolume(request *ec2.AttachVolumeInput) (*ec2.VolumeAttachment, error) { - return s.ec2.AttachVolume(request) + requestTime := time.Now() + resp, err := s.ec2.AttachVolume(request) + timeTaken := time.Since(requestTime).Seconds() + recordAwsMetric("attach_volume", timeTaken, err) + return resp, err } func (s *awsSdkEC2) DetachVolume(request *ec2.DetachVolumeInput) (*ec2.VolumeAttachment, error) { - return s.ec2.DetachVolume(request) + requestTime := time.Now() + resp, err := s.ec2.DetachVolume(request) + timeTaken := time.Since(requestTime).Seconds() + recordAwsMetric("detach_volume", timeTaken, err) + return resp, err } func (s *awsSdkEC2) DescribeVolumes(request *ec2.DescribeVolumesInput) ([]*ec2.Volume, error) { // Volumes are paged results := []*ec2.Volume{} var nextToken *string - + requestTime := time.Now() for { response, err := s.ec2.DescribeVolumes(request) if err != nil { + recordAwsMetric("describe_volume", 0, err) return nil, fmt.Errorf("error listing AWS volumes: %v", err) } @@ -630,16 +642,25 @@ func (s *awsSdkEC2) DescribeVolumes(request *ec2.DescribeVolumesInput) ([]*ec2.V } request.NextToken = nextToken } - + timeTaken := time.Since(requestTime).Seconds() + recordAwsMetric("describe_volume", timeTaken, nil) return results, nil } -func (s *awsSdkEC2) CreateVolume(request *ec2.CreateVolumeInput) (resp *ec2.Volume, err error) { - return s.ec2.CreateVolume(request) +func (s *awsSdkEC2) CreateVolume(request *ec2.CreateVolumeInput) (*ec2.Volume, error) { + requestTime := time.Now() + resp, err := s.ec2.CreateVolume(request) + timeTaken := time.Since(requestTime).Seconds() + recordAwsMetric("create_volume", timeTaken, err) + return resp, err } func (s *awsSdkEC2) DeleteVolume(request *ec2.DeleteVolumeInput) (*ec2.DeleteVolumeOutput, error) { - return s.ec2.DeleteVolume(request) + requestTime := time.Now() + resp, err := s.ec2.DeleteVolume(request) + timeTaken := time.Since(requestTime).Seconds() + recordAwsMetric("delete_volume", timeTaken, err) + return resp, err } func (s *awsSdkEC2) DescribeSubnets(request *ec2.DescribeSubnetsInput) ([]*ec2.Subnet, error) { @@ -668,7 +689,11 @@ func (s *awsSdkEC2) RevokeSecurityGroupIngress(request *ec2.RevokeSecurityGroupI } func (s *awsSdkEC2) CreateTags(request *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { - return s.ec2.CreateTags(request) + requestTime := time.Now() + resp, err := s.ec2.CreateTags(request) + timeTaken := time.Since(requestTime).Seconds() + recordAwsMetric("create_tags", timeTaken, err) + return resp, err } func (s *awsSdkEC2) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) { @@ -693,6 +718,7 @@ func (s *awsSdkEC2) ModifyInstanceAttribute(request *ec2.ModifyInstanceAttribute } func init() { + registerMetrics() cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { creds := credentials.NewChainCredentials( []credentials.Provider{ @@ -3337,3 +3363,12 @@ func setNodeDisk( } volumeMap[volumeID] = check } + +func recordAwsMetric(actionName string, timeTaken float64, err error) { + if err != nil { + awsApiErrorMetric.With(prometheus.Labels{"request": actionName}).Inc() + } else { + awsApiMetric.With(prometheus.Labels{"request": actionName}).Observe(timeTaken) + } + +} diff --git a/pkg/cloudprovider/providers/aws/aws_metrics.go b/pkg/cloudprovider/providers/aws/aws_metrics.go new file mode 100644 index 00000000000..ded8a7b5fac --- /dev/null +++ b/pkg/cloudprovider/providers/aws/aws_metrics.go @@ -0,0 +1,40 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import "github.com/prometheus/client_golang/prometheus" + +var awsApiMetric = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "cloudprovider_aws_api_request_duration_seconds", + Help: "Latency of aws api call", + }, + []string{"request"}, +) + +var awsApiErrorMetric = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cloudprovider_aws_api_request_errors", + Help: "AWS Api errors", + }, + []string{"request"}, +) + +func registerMetrics() { + prometheus.MustRegister(awsApiMetric) + prometheus.MustRegister(awsApiErrorMetric) +}