Adding vsphere Storage API Latency and Error Metrics support

fix bazel failure

pkg/cloudprovider/providers/vsphere/BUILD

@@ -12,6 +12,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "vsphere.go",
+        "vsphere_metrics.go",
         "vsphere_util.go",
     ],
     tags = ["automanaged"],
@@ -21,6 +22,7 @@ go_library(
         "//pkg/cloudprovider:go_default_library",
         "//pkg/controller:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/vmware/govmomi:go_default_library",
         "//vendor/github.com/vmware/govmomi/find:go_default_library",
         "//vendor/github.com/vmware/govmomi/object:go_default_library",

pkg/cloudprovider/providers/vsphere/vsphere.go

@@ -225,6 +225,7 @@ func readConfig(config io.Reader) (VSphereConfig, error) {
 }
 
 func init() {
+	registerMetrics()
 	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
 		cfg, err := readConfig(config)
 		if err != nil {
@@ -746,6 +747,7 @@ func cleanUpController(ctx context.Context, newSCSIController types.BaseVirtualD
 
 // Attaches given virtual disk volume to the compute running kubelet.
 func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyID string, nodeName k8stypes.NodeName) (diskID string, diskUUID string, err error) {
+	attachDiskInternal := func(vmDiskPath string, storagePolicyID string, nodeName k8stypes.NodeName) (diskID string, diskUUID string, err error) {
 		var newSCSIController types.BaseVirtualDevice
 
 		// Create context
@@ -855,8 +857,10 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyID string, nodeNam
 			deviceConfigSpec.Profile = append(deviceConfigSpec.Profile, profileSpec)
 		}
 		virtualMachineConfigSpec.DeviceChange = append(virtualMachineConfigSpec.DeviceChange, deviceConfigSpec)
+		requestTime := time.Now()
 		task, err := vm.Reconfigure(ctx, virtualMachineConfigSpec)
 		if err != nil {
+			recordvSphereMetric(api_attachvolume, requestTime, err)
 			glog.Errorf("Failed to attach the disk with storagePolicy: %+q with err - %v", storagePolicyID, err)
 			if newSCSICreated {
 				cleanUpController(ctx, newSCSIController, vmDevices, vm)
@@ -864,6 +868,7 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyID string, nodeNam
 			return "", "", err
 		}
 		err = task.Wait(ctx)
+		recordvSphereMetric(api_attachvolume, requestTime, err)
 		if err != nil {
 			glog.Errorf("Failed to attach the disk with storagePolicy: %+q with err - %v", storagePolicyID, err)
 			if newSCSICreated {
@@ -881,6 +886,11 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyID string, nodeNam
 			return "", "", err
 		}
 		return deviceName, diskUUID, nil
+	}
+	requestTime := time.Now()
+	diskID, diskUUID, err = attachDiskInternal(vmDiskPath, storagePolicyID, nodeName)
+	recordvSphereMetric(operation_attachvolume, requestTime, err)
+	return diskID, diskUUID, err
 }
 
 func getVMDiskInfo(ctx context.Context, vm *object.VirtualMachine, disk *types.VirtualDisk) (string, string, error) {
@@ -981,6 +991,7 @@ func getAvailableSCSIController(scsiControllers []*types.VirtualController) *typ
 
 // DiskIsAttached returns if disk is attached to the VM using controllers supported by the plugin.
 func (vs *VSphere) DiskIsAttached(volPath string, nodeName k8stypes.NodeName) (bool, error) {
+	diskIsAttachedInternal := func(volPath string, nodeName k8stypes.NodeName) (bool, error) {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 
@@ -1024,10 +1035,16 @@ func (vs *VSphere) DiskIsAttached(volPath string, nodeName k8stypes.NodeName) (b
 
 		attached, err := checkDiskAttached(volPath, vmDevices, dc, vs.client)
 		return attached, err
+	}
+	requestTime := time.Now()
+	isAttached, err := diskIsAttachedInternal(volPath, nodeName)
+	recordvSphereMetric(operation_diskIsAttached, requestTime, err)
+	return isAttached, err
 }
 
 // DisksAreAttached returns if disks are attached to the VM using controllers supported by the plugin.
 func (vs *VSphere) DisksAreAttached(volPaths []string, nodeName k8stypes.NodeName) (map[string]bool, error) {
+	disksAreAttachedInternal := func(volPaths []string, nodeName k8stypes.NodeName) (map[string]bool, error) {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 
@@ -1084,6 +1101,11 @@ func (vs *VSphere) DisksAreAttached(volPaths []string, nodeName k8stypes.NodeNam
 			}
 		}
 		return attached, nil
+	}
+	requestTime := time.Now()
+	attached, err := disksAreAttachedInternal(volPaths, nodeName)
+	recordvSphereMetric(operation_disksAreAttached, requestTime, err)
+	return attached, err
 }
 
 func checkDiskAttached(volPath string, vmdevices object.VirtualDeviceList, dc *object.Datacenter, client *govmomi.Client) (bool, error) {
@@ -1206,6 +1228,7 @@ func getVirtualDiskID(volPath string, vmDevices object.VirtualDeviceList, dc *ob
 
 // DetachDisk detaches given virtual disk volume from the compute running kubelet.
 func (vs *VSphere) DetachDisk(volPath string, nodeName k8stypes.NodeName) error {
+	detachDiskInternal := func(volPath string, nodeName k8stypes.NodeName) error {
 		// Create context
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
@@ -1245,17 +1268,23 @@ func (vs *VSphere) DetachDisk(volPath string, nodeName k8stypes.NodeName) error
 		}
 
 		// Detach disk from VM
+		requestTime := time.Now()
 		err = vm.RemoveDevice(ctx, true, device)
+		recordvSphereMetric(api_detachvolume, requestTime, err)
 		if err != nil {
 			return err
 		}
-
 		return nil
+	}
+	requestTime := time.Now()
+	err := detachDiskInternal(volPath, nodeName)
+	recordvSphereMetric(operation_detachvolume, requestTime, nil)
+	return err
 }
 
 // CreateVolume creates a volume of given size (in KiB).
 func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string, err error) {
-
+	createVolumeInternal := func(volumeOptions *VolumeOptions) (volumePath string, err error) {
 		var datastore string
 		var destVolPath string
 
@@ -1302,6 +1331,7 @@ func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string
 			}
 			volumeOptions.StoragePolicyID, err = pbmClient.ProfileIDByName(ctx, volumeOptions.StoragePolicyName)
 			if err != nil {
+				recordvSphereMetric(operation_createvolume_with_policy, time.Time{}, err)
 				return "", err
 			}
 
@@ -1322,6 +1352,7 @@ func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string
 			} else {
 				dsMoList, err := vs.GetCompatibleDatastoresMo(ctx, compatibilityResult)
 				if err != nil {
+					recordvSphereMetric(operation_createvolume_with_raw_vsan_policy, time.Time{}, err)
 					return "", err
 				}
 				dsMo := GetMostFreeDatastore(dsMoList)
@@ -1430,11 +1461,20 @@ func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string
 		}
 		glog.V(1).Infof("VM Disk path is %+q", destVolPath)
 		return destVolPath, nil
+	}
+	requestTime := time.Now()
+	volumePath, err = createVolumeInternal(volumeOptions)
+	recordCreateVolumeMetric(volumeOptions, requestTime, err)
+	if err != nil {
+		return "", err
+	}
+	return volumePath, nil
 }
 
 // DeleteVolume deletes a volume given volume name.
 // Also, deletes the folder where the volume resides.
 func (vs *VSphere) DeleteVolume(vmDiskPath string) error {
+	deleteVolumeInternal := func(vmDiskPath string) error {
 		// Create context
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
@@ -1477,12 +1517,20 @@ func (vs *VSphere) DeleteVolume(vmDiskPath string) error {
 
 		// Delete virtual disk
 		vmDiskPath = removeClusterFromVDiskPath(vmDiskPath)
+		requestTime := time.Now()
 		task, err := virtualDiskManager.DeleteVirtualDisk(ctx, vmDiskPath, dc)
 		if err != nil {
+			recordvSphereMetric(api_deletevolume, requestTime, err)
 			return err
 		}
-
-	return task.Wait(ctx)
+		err = task.Wait(ctx)
+		recordvSphereMetric(api_deletevolume, requestTime, err)
+		return err
+	}
+	requestTime := time.Now()
+	err := deleteVolumeInternal(vmDiskPath)
+	recordvSphereMetric(operation_deletevolume, requestTime, err)
+	return err
 }
 
 // NodeExists checks if the node with given nodeName exist.
@@ -1802,11 +1850,15 @@ func createVirtualDisk(ctx context.Context, c *govmomi.Client, dc *object.Datace
 	}
 
 	// Create virtual disk
+	requestTime := time.Now()
 	task, err := virtualDiskManager.CreateVirtualDisk(ctx, vmDiskPath, dc, vmDiskSpec)
 	if err != nil {
+		recordvSphereMetric(api_createvolume, requestTime, err)
 		return "", err
 	}
-	return vmDiskPath, task.Wait(ctx)
+	err = task.Wait(ctx)
+	recordvSphereMetric(api_createvolume, requestTime, err)
+	return vmDiskPath, err
 }
 
 // Check if the provided datastore is VSAN

pkg/cloudprovider/providers/vsphere/vsphere_metrics.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"time"
+)
+
+const (
+	api_createvolume = "CreateVolume"
+	api_deletevolume = "DeleteVolume"
+	api_attachvolume = "AttachVolume"
+	api_detachvolume = "DetachVolume"
+)
+
+const (
+	operation_deletevolume                      = "DeleteVolumeOperation"
+	operation_attachvolume                      = "AttachVolumeOperation"
+	operation_detachvolume                      = "DetachVolumeOperation"
+	operation_diskIsAttached                    = "DiskIsAttachedOperation"
+	operation_disksAreAttached                  = "DisksAreAttachedOperation"
+	operation_createvolume                      = "CreateVolumeOperation"
+	operation_createvolume_with_policy          = "CreateVolumeWithPolicyOperation"
+	operation_createvolume_with_raw_vsan_policy = "CreateVolumeWithRawVSANPolicyOperation"
+)
+
+// vsphereApiMetric is for recording latency of Single API Call.
+var vsphereApiMetric = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Name: "cloudprovider_vsphere_api_request_duration_seconds",
+		Help: "Latency of vsphere api call",
+	},
+	[]string{"request"},
+)
+
+var vsphereApiErrorMetric = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cloudprovider_vsphere_api_request_errors",
+		Help: "vsphere Api errors",
+	},
+	[]string{"request"},
+)
+
+// vsphereOperationMetric is for recording latency of vSphere Operation which invokes multiple APIs to get the task done.
+var vsphereOperationMetric = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Name: "cloudprovider_vsphere_operation_duration_seconds",
+		Help: "Latency of vsphere operation call",
+	},
+	[]string{"operation"},
+)
+
+var vsphereOperationErrorMetric = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cloudprovider_vsphere_operation_errors",
+		Help: "vsphere operation errors",
+	},
+	[]string{"operation"},
+)
+
+func registerMetrics() {
+	prometheus.MustRegister(vsphereApiMetric)
+	prometheus.MustRegister(vsphereApiErrorMetric)
+	prometheus.MustRegister(vsphereOperationMetric)
+	prometheus.MustRegister(vsphereOperationErrorMetric)
+}
+
+func recordvSphereMetric(actionName string, requestTime time.Time, err error) {
+	switch actionName {
+	case api_createvolume, api_deletevolume, api_attachvolume, api_detachvolume:
+		recordvSphereAPIMetric(actionName, requestTime, err)
+	default:
+		recordvSphereOperationMetric(actionName, requestTime, err)
+	}
+}
+
+func recordvSphereAPIMetric(actionName string, requestTime time.Time, err error) {
+	if err != nil {
+		vsphereApiErrorMetric.With(prometheus.Labels{"request": actionName}).Inc()
+	} else {
+		vsphereApiMetric.With(prometheus.Labels{"request": actionName}).Observe(calculateTimeTaken(requestTime))
+	}
+}
+
+func recordvSphereOperationMetric(actionName string, requestTime time.Time, err error) {
+	if err != nil {
+		vsphereOperationErrorMetric.With(prometheus.Labels{"operation": actionName}).Inc()
+	} else {
+		vsphereOperationMetric.With(prometheus.Labels{"operation": actionName}).Observe(calculateTimeTaken(requestTime))
+	}
+}
+
+func recordCreateVolumeMetric(volumeOptions *VolumeOptions, requestTime time.Time, err error) {
+	var actionName string
+	if volumeOptions.StoragePolicyName != "" {
+		actionName = operation_createvolume_with_policy
+	} else if volumeOptions.VSANStorageProfileData != "" {
+		actionName = operation_createvolume_with_raw_vsan_policy
+	} else {
+		actionName = operation_createvolume
+	}
+	recordvSphereMetric(actionName, requestTime, err)
+}
+
+func calculateTimeTaken(requestBeginTime time.Time) (timeTaken float64) {
+	if !requestBeginTime.IsZero() {
+		timeTaken = time.Since(requestBeginTime).Seconds()
+	} else {
+		timeTaken = 0
+	}
+	return timeTaken
+}