Implement API usage metrics for gce

This PR implements tracking of GCE API usage via prometheus metrics.

pkg/cloudprovider/providers/gce/BUILD

@@ -23,6 +23,7 @@ go_library(
         "gce_instancegroup.go",
         "gce_instances.go",
         "gce_loadbalancer.go",
+        "gce_metrics.go",
         "gce_op.go",
         "gce_routes.go",
         "gce_staticip.go",
@@ -42,10 +43,12 @@ go_library(
         "//vendor:cloud.google.com/go/compute/metadata",
         "//vendor:github.com/golang/glog",
         "//vendor:github.com/prometheus/client_golang/prometheus",
+        "//vendor:golang.org/x/net/context",
         "//vendor:golang.org/x/oauth2",
         "//vendor:golang.org/x/oauth2/google",
         "//vendor:google.golang.org/api/compute/v1",
         "//vendor:google.golang.org/api/container/v1",
+        "//vendor:google.golang.org/api/gensupport",
         "//vendor:google.golang.org/api/googleapi",
         "//vendor:gopkg.in/gcfg.v1",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",

pkg/cloudprovider/providers/gce/gce.go

@@ -20,11 +20,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"regexp"
 	"strings"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
+	"golang.org/x/net/context"
 
 	"gopkg.in/gcfg.v1"
 
@@ -38,6 +40,7 @@ import (
 	"golang.org/x/oauth2/google"
 	compute "google.golang.org/api/compute/v1"
 	container "google.golang.org/api/container/v1"
+	"google.golang.org/api/gensupport"
 )
 
 const (
@@ -100,12 +103,47 @@ type Config struct {
 	}
 }
 
+// ApiWithNamespace stores api and namespace in context
+type apiWithNamespace struct {
+	namespace string
+	apiCall   string
+}
+
 func init() {
+	registerMetrics()
 	cloudprovider.RegisterCloudProvider(
 		ProviderName,
 		func(config io.Reader) (cloudprovider.Interface, error) {
 			return newGCECloud(config)
 		})
+	gensupport.RegisterHook(trackAPILatency)
+}
+
+func trackAPILatency(ctx context.Context, req *http.Request) func(resp *http.Response) {
+	requestTime := time.Now()
+	t := ctx.Value("kube-api-namespace")
+	apiNamespace, ok := t.(apiWithNamespace)
+
+	if !ok {
+		return nil
+	}
+
+	apiResponseReceived := func(resp *http.Response) {
+		timeTaken := time.Since(requestTime).Seconds()
+		if mi, ok := gceMetricMap[apiNamespace.apiCall]; ok {
+			mi.WithLabelValues(apiNamespace.namespace).Observe(timeTaken)
+		}
+	}
+	return apiResponseReceived
+}
+
+func contextWithNamespace(namespace string, apiCall string) context.Context {
+	rootContext := context.Background()
+	apiNamespace := apiWithNamespace{
+		namespace: namespace,
+		apiCall:   apiCall,
+	}
+	return context.WithValue(rootContext, "kube-api-namespace", apiNamespace)
 }
 
 // Raw access to the underlying GCE service, probably should only be used for e2e tests

pkg/cloudprovider/providers/gce/gce_disks.go

@@ -98,8 +98,8 @@ func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOn
 		readWrite = "READ_ONLY"
 	}
 	attachedDisk := gce.convertDiskToAttachedDisk(disk, readWrite)
-
-	attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instance.Name, attachedDisk).Do()
+	dc := contextWithNamespace(diskName, "gce_attach_disk")
+	attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instance.Name, attachedDisk).Context(dc).Do()
 	if err != nil {
 		return err
 	}
@@ -122,8 +122,8 @@ func (gce *GCECloud) DetachDisk(devicePath string, nodeName types.NodeName) erro
 
 		return fmt.Errorf("error getting instance %q", instanceName)
 	}
-
-	detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Do()
+	dc := contextWithNamespace(devicePath, "gce_detach_disk")
+	detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Context(dc).Do()
 	if err != nil {
 		return err
 	}
@@ -227,8 +227,8 @@ func (gce *GCECloud) CreateDisk(name string, diskType string, zone string, sizeG
 		Description: tagsStr,
 		Type:        diskTypeUri,
 	}
-
-	createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Do()
+	dc := contextWithNamespace(name, "gce_disk_insert")
+	createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Context(dc).Do()
 	if err != nil {
 		return err
 	}
@@ -303,7 +303,8 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
 // Returns a gceDisk for the disk, if it is found in the specified zone.
 // If not found, returns (nil, nil)
 func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, error) {
-	disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Do()
+	dc := contextWithNamespace(diskName, "gce_list_disk")
+	disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Context(dc).Do()
 	if err == nil {
 		d := &gceDisk{
 			Zone: lastComponent(disk.Zone),
@@ -387,7 +388,8 @@ func (gce *GCECloud) doDeleteDisk(diskToDelete string) error {
 		return err
 	}
 
-	deleteOp, err := gce.service.Disks.Delete(gce.projectID, disk.Zone, disk.Name).Do()
+	dc := contextWithNamespace(diskToDelete, "gce_disk_delete")
+	deleteOp, err := gce.service.Disks.Delete(gce.projectID, disk.Zone, disk.Name).Context(dc).Do()
 	if err != nil {
 		return err
 	}

pkg/cloudprovider/providers/gce/gce_instances.go

@@ -283,7 +283,8 @@ func (gce *GCECloud) getInstanceByName(name string) (*gceInstance, error) {
 	// Avoid changing behaviour when not managing multiple zones
 	for _, zone := range gce.managedZones {
 		name = canonicalizeInstanceName(name)
-		res, err := gce.service.Instances.Get(gce.projectID, zone, name).Do()
+		dc := contextWithNamespace(name, "gce_instance_list")
+		res, err := gce.service.Instances.Get(gce.projectID, zone, name).Context(dc).Do()
 		if err != nil {
 			glog.Errorf("getInstanceByName: failed to get instance %s; err: %v", name, err)
 

pkg/cloudprovider/providers/gce/gce_metrics.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var gceMetricMap = map[string]*prometheus.HistogramVec{
+	"gce_instance_list": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "gce_instance_list_duration_seconds",
+			Help: "Latency of instance listing calls",
+		},
+		[]string{"namespace"},
+	),
+	"gce_disk_insert": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "gce_disk_insert_duration_seconds",
+			Help: "Latency of disk insert calls",
+		},
+		[]string{"namespace"},
+	),
+	"gce_disk_delete": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "gce_disk_delete_duration_seconds",
+			Help: "Latency of disk delete calls",
+		},
+		[]string{"namespace"},
+	),
+	"gce_attach_disk": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "gce_attach_disk_duration_seconds",
+			Help: "Latency of attach disk calls",
+		},
+		[]string{"namespace"},
+	),
+	"gce_detach_disk": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "gce_detach_disk_duration_seconds",
+			Help: "Latency of detach disk calls",
+		},
+		[]string{"namespace"},
+	),
+	"gce_list_disk": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "gce_list_disk_duration_seconds",
+			Help: "Latency of list disk calls",
+		},
+		[]string{"namespace"},
+	),
+}
+
+func registerMetrics() {
+	for _, metric := range gceMetricMap {
+		prometheus.MustRegister(metric)
+	}
+}