Implement API usage metrics for gce

This PR implements tracking of GCE API usage via prometheus metrics.
This commit is contained in:
Hemant Kumar 2017-03-28 16:22:55 -04:00
parent c01baaf54f
commit c4aaf47282
5 changed files with 123 additions and 9 deletions

View File

@ -23,6 +23,7 @@ go_library(
"gce_instancegroup.go",
"gce_instances.go",
"gce_loadbalancer.go",
"gce_metrics.go",
"gce_op.go",
"gce_routes.go",
"gce_staticip.go",
@ -42,10 +43,12 @@ go_library(
"//vendor:cloud.google.com/go/compute/metadata",
"//vendor:github.com/golang/glog",
"//vendor:github.com/prometheus/client_golang/prometheus",
"//vendor:golang.org/x/net/context",
"//vendor:golang.org/x/oauth2",
"//vendor:golang.org/x/oauth2/google",
"//vendor:google.golang.org/api/compute/v1",
"//vendor:google.golang.org/api/container/v1",
"//vendor:google.golang.org/api/gensupport",
"//vendor:google.golang.org/api/googleapi",
"//vendor:gopkg.in/gcfg.v1",
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",

View File

@ -20,11 +20,13 @@ import (
"errors"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
"cloud.google.com/go/compute/metadata"
"golang.org/x/net/context"
"gopkg.in/gcfg.v1"
@ -38,6 +40,7 @@ import (
"golang.org/x/oauth2/google"
compute "google.golang.org/api/compute/v1"
container "google.golang.org/api/container/v1"
"google.golang.org/api/gensupport"
)
const (
@ -100,12 +103,47 @@ type Config struct {
}
}
// ApiWithNamespace stores api and namespace in context
type apiWithNamespace struct {
namespace string
apiCall string
}
func init() {
registerMetrics()
cloudprovider.RegisterCloudProvider(
ProviderName,
func(config io.Reader) (cloudprovider.Interface, error) {
return newGCECloud(config)
})
gensupport.RegisterHook(trackAPILatency)
}
func trackAPILatency(ctx context.Context, req *http.Request) func(resp *http.Response) {
requestTime := time.Now()
t := ctx.Value("kube-api-namespace")
apiNamespace, ok := t.(apiWithNamespace)
if !ok {
return nil
}
apiResponseReceived := func(resp *http.Response) {
timeTaken := time.Since(requestTime).Seconds()
if mi, ok := gceMetricMap[apiNamespace.apiCall]; ok {
mi.WithLabelValues(apiNamespace.namespace).Observe(timeTaken)
}
}
return apiResponseReceived
}
func contextWithNamespace(namespace string, apiCall string) context.Context {
rootContext := context.Background()
apiNamespace := apiWithNamespace{
namespace: namespace,
apiCall: apiCall,
}
return context.WithValue(rootContext, "kube-api-namespace", apiNamespace)
}
// Raw access to the underlying GCE service, probably should only be used for e2e tests

View File

@ -98,8 +98,8 @@ func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOn
readWrite = "READ_ONLY"
}
attachedDisk := gce.convertDiskToAttachedDisk(disk, readWrite)
attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instance.Name, attachedDisk).Do()
dc := contextWithNamespace(diskName, "gce_attach_disk")
attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instance.Name, attachedDisk).Context(dc).Do()
if err != nil {
return err
}
@ -122,8 +122,8 @@ func (gce *GCECloud) DetachDisk(devicePath string, nodeName types.NodeName) erro
return fmt.Errorf("error getting instance %q", instanceName)
}
detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Do()
dc := contextWithNamespace(devicePath, "gce_detach_disk")
detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Context(dc).Do()
if err != nil {
return err
}
@ -227,8 +227,8 @@ func (gce *GCECloud) CreateDisk(name string, diskType string, zone string, sizeG
Description: tagsStr,
Type: diskTypeUri,
}
createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Do()
dc := contextWithNamespace(name, "gce_disk_insert")
createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Context(dc).Do()
if err != nil {
return err
}
@ -303,7 +303,8 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
// Returns a gceDisk for the disk, if it is found in the specified zone.
// If not found, returns (nil, nil)
func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, error) {
disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Do()
dc := contextWithNamespace(diskName, "gce_list_disk")
disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Context(dc).Do()
if err == nil {
d := &gceDisk{
Zone: lastComponent(disk.Zone),
@ -387,7 +388,8 @@ func (gce *GCECloud) doDeleteDisk(diskToDelete string) error {
return err
}
deleteOp, err := gce.service.Disks.Delete(gce.projectID, disk.Zone, disk.Name).Do()
dc := contextWithNamespace(diskToDelete, "gce_disk_delete")
deleteOp, err := gce.service.Disks.Delete(gce.projectID, disk.Zone, disk.Name).Context(dc).Do()
if err != nil {
return err
}

View File

@ -283,7 +283,8 @@ func (gce *GCECloud) getInstanceByName(name string) (*gceInstance, error) {
// Avoid changing behaviour when not managing multiple zones
for _, zone := range gce.managedZones {
name = canonicalizeInstanceName(name)
res, err := gce.service.Instances.Get(gce.projectID, zone, name).Do()
dc := contextWithNamespace(name, "gce_instance_list")
res, err := gce.service.Instances.Get(gce.projectID, zone, name).Context(dc).Do()
if err != nil {
glog.Errorf("getInstanceByName: failed to get instance %s; err: %v", name, err)

View File

@ -0,0 +1,70 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gce
import "github.com/prometheus/client_golang/prometheus"
var gceMetricMap = map[string]*prometheus.HistogramVec{
"gce_instance_list": prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gce_instance_list_duration_seconds",
Help: "Latency of instance listing calls",
},
[]string{"namespace"},
),
"gce_disk_insert": prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gce_disk_insert_duration_seconds",
Help: "Latency of disk insert calls",
},
[]string{"namespace"},
),
"gce_disk_delete": prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gce_disk_delete_duration_seconds",
Help: "Latency of disk delete calls",
},
[]string{"namespace"},
),
"gce_attach_disk": prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gce_attach_disk_duration_seconds",
Help: "Latency of attach disk calls",
},
[]string{"namespace"},
),
"gce_detach_disk": prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gce_detach_disk_duration_seconds",
Help: "Latency of detach disk calls",
},
[]string{"namespace"},
),
"gce_list_disk": prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gce_list_disk_duration_seconds",
Help: "Latency of list disk calls",
},
[]string{"namespace"},
),
}
func registerMetrics() {
for _, metric := range gceMetricMap {
prometheus.MustRegister(metric)
}
}