From c4aaf47282fe6bf1c370eb6d7ef958045e2384b3 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 28 Mar 2017 16:22:55 -0400 Subject: [PATCH] Implement API usage metrics for gce This PR implements tracking of GCE API usage via prometheus metrics. --- pkg/cloudprovider/providers/gce/BUILD | 3 + pkg/cloudprovider/providers/gce/gce.go | 38 ++++++++++ pkg/cloudprovider/providers/gce/gce_disks.go | 18 ++--- .../providers/gce/gce_instances.go | 3 +- .../providers/gce/gce_metrics.go | 70 +++++++++++++++++++ 5 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 pkg/cloudprovider/providers/gce/gce_metrics.go diff --git a/pkg/cloudprovider/providers/gce/BUILD b/pkg/cloudprovider/providers/gce/BUILD index e10732dabc9..85f22652a99 100644 --- a/pkg/cloudprovider/providers/gce/BUILD +++ b/pkg/cloudprovider/providers/gce/BUILD @@ -23,6 +23,7 @@ go_library( "gce_instancegroup.go", "gce_instances.go", "gce_loadbalancer.go", + "gce_metrics.go", "gce_op.go", "gce_routes.go", "gce_staticip.go", @@ -42,10 +43,12 @@ go_library( "//vendor:cloud.google.com/go/compute/metadata", "//vendor:github.com/golang/glog", "//vendor:github.com/prometheus/client_golang/prometheus", + "//vendor:golang.org/x/net/context", "//vendor:golang.org/x/oauth2", "//vendor:golang.org/x/oauth2/google", "//vendor:google.golang.org/api/compute/v1", "//vendor:google.golang.org/api/container/v1", + "//vendor:google.golang.org/api/gensupport", "//vendor:google.golang.org/api/googleapi", "//vendor:gopkg.in/gcfg.v1", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index f9253856dac..8355d581f42 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -20,11 +20,13 @@ import ( "errors" "fmt" "io" + "net/http" "regexp" "strings" "time" "cloud.google.com/go/compute/metadata" + "golang.org/x/net/context" "gopkg.in/gcfg.v1" @@ -38,6 +40,7 @@ import ( "golang.org/x/oauth2/google" compute "google.golang.org/api/compute/v1" container "google.golang.org/api/container/v1" + "google.golang.org/api/gensupport" ) const ( @@ -100,12 +103,47 @@ type Config struct { } } +// ApiWithNamespace stores api and namespace in context +type apiWithNamespace struct { + namespace string + apiCall string +} + func init() { + registerMetrics() cloudprovider.RegisterCloudProvider( ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { return newGCECloud(config) }) + gensupport.RegisterHook(trackAPILatency) +} + +func trackAPILatency(ctx context.Context, req *http.Request) func(resp *http.Response) { + requestTime := time.Now() + t := ctx.Value("kube-api-namespace") + apiNamespace, ok := t.(apiWithNamespace) + + if !ok { + return nil + } + + apiResponseReceived := func(resp *http.Response) { + timeTaken := time.Since(requestTime).Seconds() + if mi, ok := gceMetricMap[apiNamespace.apiCall]; ok { + mi.WithLabelValues(apiNamespace.namespace).Observe(timeTaken) + } + } + return apiResponseReceived +} + +func contextWithNamespace(namespace string, apiCall string) context.Context { + rootContext := context.Background() + apiNamespace := apiWithNamespace{ + namespace: namespace, + apiCall: apiCall, + } + return context.WithValue(rootContext, "kube-api-namespace", apiNamespace) } // Raw access to the underlying GCE service, probably should only be used for e2e tests diff --git a/pkg/cloudprovider/providers/gce/gce_disks.go b/pkg/cloudprovider/providers/gce/gce_disks.go index 159d5755a1d..12c0cca1d18 100644 --- a/pkg/cloudprovider/providers/gce/gce_disks.go +++ b/pkg/cloudprovider/providers/gce/gce_disks.go @@ -98,8 +98,8 @@ func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOn readWrite = "READ_ONLY" } attachedDisk := gce.convertDiskToAttachedDisk(disk, readWrite) - - attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instance.Name, attachedDisk).Do() + dc := contextWithNamespace(diskName, "gce_attach_disk") + attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instance.Name, attachedDisk).Context(dc).Do() if err != nil { return err } @@ -122,8 +122,8 @@ func (gce *GCECloud) DetachDisk(devicePath string, nodeName types.NodeName) erro return fmt.Errorf("error getting instance %q", instanceName) } - - detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Do() + dc := contextWithNamespace(devicePath, "gce_detach_disk") + detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Context(dc).Do() if err != nil { return err } @@ -227,8 +227,8 @@ func (gce *GCECloud) CreateDisk(name string, diskType string, zone string, sizeG Description: tagsStr, Type: diskTypeUri, } - - createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Do() + dc := contextWithNamespace(name, "gce_disk_insert") + createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Context(dc).Do() if err != nil { return err } @@ -303,7 +303,8 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st // Returns a gceDisk for the disk, if it is found in the specified zone. // If not found, returns (nil, nil) func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, error) { - disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Do() + dc := contextWithNamespace(diskName, "gce_list_disk") + disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Context(dc).Do() if err == nil { d := &gceDisk{ Zone: lastComponent(disk.Zone), @@ -387,7 +388,8 @@ func (gce *GCECloud) doDeleteDisk(diskToDelete string) error { return err } - deleteOp, err := gce.service.Disks.Delete(gce.projectID, disk.Zone, disk.Name).Do() + dc := contextWithNamespace(diskToDelete, "gce_disk_delete") + deleteOp, err := gce.service.Disks.Delete(gce.projectID, disk.Zone, disk.Name).Context(dc).Do() if err != nil { return err } diff --git a/pkg/cloudprovider/providers/gce/gce_instances.go b/pkg/cloudprovider/providers/gce/gce_instances.go index 1813d43b040..2ad33a5d493 100644 --- a/pkg/cloudprovider/providers/gce/gce_instances.go +++ b/pkg/cloudprovider/providers/gce/gce_instances.go @@ -283,7 +283,8 @@ func (gce *GCECloud) getInstanceByName(name string) (*gceInstance, error) { // Avoid changing behaviour when not managing multiple zones for _, zone := range gce.managedZones { name = canonicalizeInstanceName(name) - res, err := gce.service.Instances.Get(gce.projectID, zone, name).Do() + dc := contextWithNamespace(name, "gce_instance_list") + res, err := gce.service.Instances.Get(gce.projectID, zone, name).Context(dc).Do() if err != nil { glog.Errorf("getInstanceByName: failed to get instance %s; err: %v", name, err) diff --git a/pkg/cloudprovider/providers/gce/gce_metrics.go b/pkg/cloudprovider/providers/gce/gce_metrics.go new file mode 100644 index 00000000000..2c97a661cba --- /dev/null +++ b/pkg/cloudprovider/providers/gce/gce_metrics.go @@ -0,0 +1,70 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import "github.com/prometheus/client_golang/prometheus" + +var gceMetricMap = map[string]*prometheus.HistogramVec{ + "gce_instance_list": prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gce_instance_list_duration_seconds", + Help: "Latency of instance listing calls", + }, + []string{"namespace"}, + ), + "gce_disk_insert": prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gce_disk_insert_duration_seconds", + Help: "Latency of disk insert calls", + }, + []string{"namespace"}, + ), + "gce_disk_delete": prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gce_disk_delete_duration_seconds", + Help: "Latency of disk delete calls", + }, + []string{"namespace"}, + ), + "gce_attach_disk": prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gce_attach_disk_duration_seconds", + Help: "Latency of attach disk calls", + }, + []string{"namespace"}, + ), + "gce_detach_disk": prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gce_detach_disk_duration_seconds", + Help: "Latency of detach disk calls", + }, + []string{"namespace"}, + ), + "gce_list_disk": prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "gce_list_disk_duration_seconds", + Help: "Latency of list disk calls", + }, + []string{"namespace"}, + ), +} + +func registerMetrics() { + for _, metric := range gceMetricMap { + prometheus.MustRegister(metric) + } +}