From 18852c58c17a6dcd75912057c0e113b469bb2ced Mon Sep 17 00:00:00 2001 From: NickrenREN Date: Wed, 17 May 2017 17:37:43 +0800 Subject: [PATCH] Recording openstack metrics add openstack operation metrics Add support for emitting metrics from openstack cloudprovider about storage operations. --- pkg/cloudprovider/providers/openstack/BUILD | 2 + .../providers/openstack/openstack.go | 7 +-- .../providers/openstack/openstack_metrics.go | 50 +++++++++++++++++++ .../providers/openstack/openstack_volumes.go | 40 ++++++++++++++- 4 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 pkg/cloudprovider/providers/openstack/openstack_metrics.go diff --git a/pkg/cloudprovider/providers/openstack/BUILD b/pkg/cloudprovider/providers/openstack/BUILD index 53f86b4ee89..de95f1dcee5 100644 --- a/pkg/cloudprovider/providers/openstack/BUILD +++ b/pkg/cloudprovider/providers/openstack/BUILD @@ -15,6 +15,7 @@ go_library( "openstack.go", "openstack_instances.go", "openstack_loadbalancer.go", + "openstack_metrics.go", "openstack_routes.go", "openstack_volumes.go", ], @@ -53,6 +54,7 @@ go_library( "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports:go_default_library", "//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library", "//vendor/github.com/mitchellh/mapstructure:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 4ad0bfd772e..9fa26e2f6b5 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -53,11 +53,6 @@ var ErrNotFound = errors.New("Failed to find object") var ErrMultipleResults = errors.New("Multiple results where only one expected") var ErrNoAddressFound = errors.New("No address found for host") -const ( - MiB = 1024 * 1024 - GB = 1000 * 1000 * 1000 -) - // encoding.TextUnmarshaler interface for time.Duration type MyDuration struct { time.Duration @@ -131,6 +126,8 @@ type Config struct { } func init() { + RegisterMetrics() + cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { cfg, err := readConfig(config) if err != nil { diff --git a/pkg/cloudprovider/providers/openstack/openstack_metrics.go b/pkg/cloudprovider/providers/openstack/openstack_metrics.go new file mode 100644 index 00000000000..75dc398cb5b --- /dev/null +++ b/pkg/cloudprovider/providers/openstack/openstack_metrics.go @@ -0,0 +1,50 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import "github.com/prometheus/client_golang/prometheus" + +const ( + OpenstackSubsystem = "openstack" + OpenstackOperationKey = "cloudprovider_openstack_api_request_duration_seconds" + OpenstackOperationErrorKey = "cloudprovider_openstack_api_request_errors" +) + +var ( + OpenstackOperationsLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: OpenstackSubsystem, + Name: OpenstackOperationKey, + Help: "Latency of openstack api call", + }, + []string{"request"}, + ) + + OpenstackApiRequestErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: OpenstackSubsystem, + Name: OpenstackOperationErrorKey, + Help: "Cumulative number of openstack Api call errors", + }, + []string{"request"}, + ) +) + +func RegisterMetrics() { + prometheus.MustRegister(OpenstackOperationsLatency) + prometheus.MustRegister(OpenstackApiRequestErrors) +} diff --git a/pkg/cloudprovider/providers/openstack/openstack_volumes.go b/pkg/cloudprovider/providers/openstack/openstack_volumes.go index 91284650980..ee6baf1d3ee 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_volumes.go +++ b/pkg/cloudprovider/providers/openstack/openstack_volumes.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "path" "strings" + "time" k8s_volume "k8s.io/kubernetes/pkg/volume" @@ -31,6 +32,7 @@ import ( volumes_v2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes" "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach" "github.com/gophercloud/gophercloud/pagination" + "github.com/prometheus/client_golang/prometheus" "github.com/golang/glog" ) @@ -82,6 +84,8 @@ const ( ) func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, error) { + startTime := time.Now() + create_opts := volumes_v1.CreateOpts{ Name: opts.Name, Size: opts.Size, @@ -91,6 +95,8 @@ func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, e } vol, err := volumes_v1.Create(volumes.blockstorage, create_opts).Extract() + timeTaken := time.Since(startTime).Seconds() + recordOpenstackOperationMetric("create_v1_volume", timeTaken, err) if err != nil { return "", "", err } @@ -98,6 +104,8 @@ func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, e } func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, error) { + startTime := time.Now() + create_opts := volumes_v2.CreateOpts{ Name: opts.Name, Size: opts.Size, @@ -107,6 +115,8 @@ func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, e } vol, err := volumes_v2.Create(volumes.blockstorage, create_opts).Extract() + timeTaken := time.Since(startTime).Seconds() + recordOpenstackOperationMetric("create_v2_volume", timeTaken, err) if err != nil { return "", "", err } @@ -116,6 +126,7 @@ func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, e func (volumes *VolumesV1) getVolume(diskName string) (Volume, error) { var volume_v1 volumes_v1.Volume var volume Volume + startTime := time.Now() err := volumes_v1.List(volumes.blockstorage, nil).EachPage(func(page pagination.Page) (bool, error) { vols, err := volumes_v1.ExtractVolumes(page) if err != nil { @@ -134,6 +145,8 @@ func (volumes *VolumesV1) getVolume(diskName string) (Volume, error) { errmsg := fmt.Sprintf("Unable to find disk: %s", diskName) return false, errors.New(errmsg) }) + timeTaken := time.Since(startTime).Seconds() + recordOpenstackOperationMetric("get_v1_volume", timeTaken, err) if err != nil { glog.Errorf("Error occurred getting volume: %s", diskName) return volume, err @@ -154,6 +167,7 @@ func (volumes *VolumesV1) getVolume(diskName string) (Volume, error) { func (volumes *VolumesV2) getVolume(diskName string) (Volume, error) { var volume_v2 volumes_v2.Volume var volume Volume + startTime := time.Now() err := volumes_v2.List(volumes.blockstorage, nil).EachPage(func(page pagination.Page) (bool, error) { vols, err := volumes_v2.ExtractVolumes(page) if err != nil { @@ -172,6 +186,8 @@ func (volumes *VolumesV2) getVolume(diskName string) (Volume, error) { errmsg := fmt.Sprintf("Unable to find disk: %s", diskName) return false, errors.New(errmsg) }) + timeTaken := time.Since(startTime).Seconds() + recordOpenstackOperationMetric("get_v2_volume", timeTaken, err) if err != nil { glog.Errorf("Error occurred getting volume: %s", diskName) return volume, err @@ -190,19 +206,26 @@ func (volumes *VolumesV2) getVolume(diskName string) (Volume, error) { } func (volumes *VolumesV1) deleteVolume(volumeName string) error { - + startTime := time.Now() err := volumes_v1.Delete(volumes.blockstorage, volumeName).ExtractErr() + timeTaken := time.Since(startTime).Seconds() + recordOpenstackOperationMetric("delete_v1_volume", timeTaken, err) if err != nil { glog.Errorf("Cannot delete volume %s: %v", volumeName, err) } + return err } func (volumes *VolumesV2) deleteVolume(volumeName string) error { + startTime := time.Now() err := volumes_v2.Delete(volumes.blockstorage, volumeName).ExtractErr() + timeTaken := time.Since(startTime).Seconds() + recordOpenstackOperationMetric("delete_v2_volume", timeTaken, err) if err != nil { glog.Errorf("Cannot delete volume %s: %v", volumeName, err) } + return err } @@ -253,10 +276,13 @@ func (os *OpenStack) AttachDisk(instanceID string, diskName string) (string, err } } + startTime := time.Now() // add read only flag here if possible spothanis _, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{ VolumeID: volume.ID, }).Extract() + timeTaken := time.Since(startTime).Seconds() + recordOpenstackOperationMetric("attach_disk", timeTaken, err) if err != nil { glog.Errorf("Failed to attach %s volume to %s compute: %v", diskName, instanceID, err) return "", err @@ -288,9 +314,12 @@ func (os *OpenStack) DetachDisk(instanceID string, partialDiskId string) error { glog.Errorf(errMsg) return errors.New(errMsg) } else { + startTime := time.Now() // This is a blocking call and effects kubelet's performance directly. // We should consider kicking it out into a separate routine, if it is bad. err = volumeattach.Delete(cClient, instanceID, volume.ID).ExtractErr() + timeTaken := time.Since(startTime).Seconds() + recordOpenstackOperationMetric("detach_disk", timeTaken, err) if err != nil { glog.Errorf("Failed to delete volume %s from compute %s attached %v", volume.ID, instanceID, err) return err @@ -458,3 +487,12 @@ func (os *OpenStack) diskIsUsed(diskName string) (bool, error) { func (os *OpenStack) ShouldTrustDevicePath() bool { return os.bsOpts.TrustDevicePath } + +// recordOpenstackOperationMetric records openstack operation metrics +func recordOpenstackOperationMetric(operation string, timeTaken float64, err error) { + if err != nil { + OpenstackApiRequestErrors.With(prometheus.Labels{"request": operation}).Inc() + } else { + OpenstackOperationsLatency.With(prometheus.Labels{"request": operation}).Observe(timeTaken) + } +}