github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 07:20:13 +00:00
Code Issues Projects Releases Wiki Activity

Recording openstack metrics

Browse Source 
add openstack operation metrics
Add support for emitting metrics from openstack cloudprovider about storage operations.
This commit is contained in:
NickrenREN 2017-05-17 17:37:43 +08:00
parent 7f183142de
commit 18852c58c1
4 changed files with 93 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/cloudprovider/providers/openstack/BUILD
View File

@ -15,6 +15,7 @@ go_library(
"openstack.go",
"openstack_instances.go",
"openstack_loadbalancer.go",
"openstack_metrics.go",
"openstack_routes.go",
"openstack_volumes.go",
],
@ -53,6 +54,7 @@ go_library(
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library",
"//vendor/github.com/mitchellh/mapstructure:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",

7
pkg/cloudprovider/providers/openstack/openstack.go
View File

@ -53,11 +53,6 @@ var ErrNotFound = errors.New("Failed to find object")
var ErrMultipleResults = errors.New("Multiple results where only one expected")
var ErrNoAddressFound = errors.New("No address found for host")
const (
MiB = 1024 * 1024
GB = 1000 * 1000 * 1000
)
// encoding.TextUnmarshaler interface for time.Duration
type MyDuration struct {
time.Duration
@ -131,6 +126,8 @@ type Config struct {
}
func init() {
RegisterMetrics()
cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
cfg, err := readConfig(config)
if err != nil {

50
pkg/cloudprovider/providers/openstack/openstack_metrics.go Normal file
View File

@ -0,0 +1,50 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import "github.com/prometheus/client_golang/prometheus"
const (
OpenstackSubsystem = "openstack"
OpenstackOperationKey = "cloudprovider_openstack_api_request_duration_seconds"
OpenstackOperationErrorKey = "cloudprovider_openstack_api_request_errors"
)
var (
OpenstackOperationsLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Subsystem: OpenstackSubsystem,
Name: OpenstackOperationKey,
Help: "Latency of openstack api call",
},
[]string{"request"},
)
OpenstackApiRequestErrors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: OpenstackSubsystem,
Name: OpenstackOperationErrorKey,
Help: "Cumulative number of openstack Api call errors",
},
[]string{"request"},
)
)
func RegisterMetrics() {
prometheus.MustRegister(OpenstackOperationsLatency)
prometheus.MustRegister(OpenstackApiRequestErrors)
}

40
pkg/cloudprovider/providers/openstack/openstack_volumes.go
View File

@ -22,6 +22,7 @@ import (
"io/ioutil"
"path"
"strings"
"time"
k8s_volume "k8s.io/kubernetes/pkg/volume"
@ -31,6 +32,7 @@ import (
volumes_v2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
"github.com/gophercloud/gophercloud/pagination"
"github.com/prometheus/client_golang/prometheus"
"github.com/golang/glog"
)
@ -82,6 +84,8 @@ const (
)
func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, error) {
startTime := time.Now()
create_opts := volumes_v1.CreateOpts{
Name: opts.Name,
Size: opts.Size,
@ -91,6 +95,8 @@ func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, e
}
vol, err := volumes_v1.Create(volumes.blockstorage, create_opts).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("create_v1_volume", timeTaken, err)
if err != nil {
return "", "", err
}
@ -98,6 +104,8 @@ func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, e
}
func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, error) {
startTime := time.Now()
create_opts := volumes_v2.CreateOpts{
Name: opts.Name,
Size: opts.Size,
@ -107,6 +115,8 @@ func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, e
}
vol, err := volumes_v2.Create(volumes.blockstorage, create_opts).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("create_v2_volume", timeTaken, err)
if err != nil {
return "", "", err
}
@ -116,6 +126,7 @@ func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, e
func (volumes *VolumesV1) getVolume(diskName string) (Volume, error) {
var volume_v1 volumes_v1.Volume
var volume Volume
startTime := time.Now()
err := volumes_v1.List(volumes.blockstorage, nil).EachPage(func(page pagination.Page) (bool, error) {
vols, err := volumes_v1.ExtractVolumes(page)
if err != nil {
@ -134,6 +145,8 @@ func (volumes *VolumesV1) getVolume(diskName string) (Volume, error) {
errmsg := fmt.Sprintf("Unable to find disk: %s", diskName)
return false, errors.New(errmsg)
})
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("get_v1_volume", timeTaken, err)
if err != nil {
glog.Errorf("Error occurred getting volume: %s", diskName)
return volume, err
@ -154,6 +167,7 @@ func (volumes *VolumesV1) getVolume(diskName string) (Volume, error) {
func (volumes *VolumesV2) getVolume(diskName string) (Volume, error) {
var volume_v2 volumes_v2.Volume
var volume Volume
startTime := time.Now()
err := volumes_v2.List(volumes.blockstorage, nil).EachPage(func(page pagination.Page) (bool, error) {
vols, err := volumes_v2.ExtractVolumes(page)
if err != nil {
@ -172,6 +186,8 @@ func (volumes *VolumesV2) getVolume(diskName string) (Volume, error) {
errmsg := fmt.Sprintf("Unable to find disk: %s", diskName)
return false, errors.New(errmsg)
})
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("get_v2_volume", timeTaken, err)
if err != nil {
glog.Errorf("Error occurred getting volume: %s", diskName)
return volume, err
@ -190,19 +206,26 @@ func (volumes *VolumesV2) getVolume(diskName string) (Volume, error) {
}
func (volumes *VolumesV1) deleteVolume(volumeName string) error {
startTime := time.Now()
err := volumes_v1.Delete(volumes.blockstorage, volumeName).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("delete_v1_volume", timeTaken, err)
if err != nil {
glog.Errorf("Cannot delete volume %s: %v", volumeName, err)
}
return err
}
func (volumes *VolumesV2) deleteVolume(volumeName string) error {
startTime := time.Now()
err := volumes_v2.Delete(volumes.blockstorage, volumeName).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("delete_v2_volume", timeTaken, err)
if err != nil {
glog.Errorf("Cannot delete volume %s: %v", volumeName, err)
}
return err
}
@ -253,10 +276,13 @@ func (os *OpenStack) AttachDisk(instanceID string, diskName string) (string, err
}
}
startTime := time.Now()
// add read only flag here if possible spothanis
_, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{
VolumeID: volume.ID,
}).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("attach_disk", timeTaken, err)
if err != nil {
glog.Errorf("Failed to attach %s volume to %s compute: %v", diskName, instanceID, err)
return "", err
@ -288,9 +314,12 @@ func (os *OpenStack) DetachDisk(instanceID string, partialDiskId string) error {
glog.Errorf(errMsg)
return errors.New(errMsg)
} else {
startTime := time.Now()
// This is a blocking call and effects kubelet's performance directly.
// We should consider kicking it out into a separate routine, if it is bad.
err = volumeattach.Delete(cClient, instanceID, volume.ID).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("detach_disk", timeTaken, err)
if err != nil {
glog.Errorf("Failed to delete volume %s from compute %s attached %v", volume.ID, instanceID, err)
return err
@ -458,3 +487,12 @@ func (os *OpenStack) diskIsUsed(diskName string) (bool, error) {
func (os *OpenStack) ShouldTrustDevicePath() bool {
return os.bsOpts.TrustDevicePath
}
// recordOpenstackOperationMetric records openstack operation metrics
func recordOpenstackOperationMetric(operation string, timeTaken float64, err error) {
if err != nil {
OpenstackApiRequestErrors.With(prometheus.Labels{"request": operation}).Inc()
} else {
OpenstackOperationsLatency.With(prometheus.Labels{"request": operation}).Observe(timeTaken)
}
}