mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 11:21:47 +00:00
Add metrics for attachable volumes in use
This commit is contained in:
parent
ee2e11a0d4
commit
8d46912e7f
@ -14,6 +14,7 @@ go_library(
|
|||||||
"//pkg/cloudprovider:go_default_library",
|
"//pkg/cloudprovider:go_default_library",
|
||||||
"//pkg/controller:go_default_library",
|
"//pkg/controller:go_default_library",
|
||||||
"//pkg/controller/volume/attachdetach/cache:go_default_library",
|
"//pkg/controller/volume/attachdetach/cache:go_default_library",
|
||||||
|
"//pkg/controller/volume/attachdetach/metrics:go_default_library",
|
||||||
"//pkg/controller/volume/attachdetach/populator:go_default_library",
|
"//pkg/controller/volume/attachdetach/populator:go_default_library",
|
||||||
"//pkg/controller/volume/attachdetach/reconciler:go_default_library",
|
"//pkg/controller/volume/attachdetach/reconciler:go_default_library",
|
||||||
"//pkg/controller/volume/attachdetach/statusupdater:go_default_library",
|
"//pkg/controller/volume/attachdetach/statusupdater:go_default_library",
|
||||||
@ -69,6 +70,7 @@ filegroup(
|
|||||||
srcs = [
|
srcs = [
|
||||||
":package-srcs",
|
":package-srcs",
|
||||||
"//pkg/controller/volume/attachdetach/cache:all-srcs",
|
"//pkg/controller/volume/attachdetach/cache:all-srcs",
|
||||||
|
"//pkg/controller/volume/attachdetach/metrics:all-srcs",
|
||||||
"//pkg/controller/volume/attachdetach/populator:all-srcs",
|
"//pkg/controller/volume/attachdetach/populator:all-srcs",
|
||||||
"//pkg/controller/volume/attachdetach/reconciler:all-srcs",
|
"//pkg/controller/volume/attachdetach/reconciler:all-srcs",
|
||||||
"//pkg/controller/volume/attachdetach/statusupdater:all-srcs",
|
"//pkg/controller/volume/attachdetach/statusupdater:all-srcs",
|
||||||
|
@ -39,6 +39,7 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/cloudprovider"
|
"k8s.io/kubernetes/pkg/cloudprovider"
|
||||||
"k8s.io/kubernetes/pkg/controller"
|
"k8s.io/kubernetes/pkg/controller"
|
||||||
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
|
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
|
||||||
|
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/metrics"
|
||||||
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/populator"
|
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/populator"
|
||||||
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler"
|
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler"
|
||||||
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater"
|
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater"
|
||||||
@ -273,6 +274,7 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) {
|
|||||||
}
|
}
|
||||||
go adc.reconciler.Run(stopCh)
|
go adc.reconciler.Run(stopCh)
|
||||||
go adc.desiredStateOfWorldPopulator.Run(stopCh)
|
go adc.desiredStateOfWorldPopulator.Run(stopCh)
|
||||||
|
metrics.Register(adc.pvcLister, adc.pvLister, adc.podLister, &adc.volumePluginMgr)
|
||||||
|
|
||||||
<-stopCh
|
<-stopCh
|
||||||
}
|
}
|
||||||
|
46
pkg/controller/volume/attachdetach/metrics/BUILD
Normal file
46
pkg/controller/volume/attachdetach/metrics/BUILD
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||||
|
|
||||||
|
go_library(
|
||||||
|
name = "go_default_library",
|
||||||
|
srcs = ["metrics.go"],
|
||||||
|
importpath = "k8s.io/kubernetes/pkg/controller/volume/attachdetach/metrics",
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [
|
||||||
|
"//pkg/controller/volume/attachdetach/util:go_default_library",
|
||||||
|
"//pkg/volume:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
|
||||||
|
"//vendor/github.com/golang/glog:go_default_library",
|
||||||
|
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_test(
|
||||||
|
name = "go_default_test",
|
||||||
|
srcs = ["metrics_test.go"],
|
||||||
|
embed = [":go_default_library"],
|
||||||
|
deps = [
|
||||||
|
"//pkg/controller:go_default_library",
|
||||||
|
"//pkg/volume/testing:go_default_library",
|
||||||
|
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/informers:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "package-srcs",
|
||||||
|
srcs = glob(["**"]),
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "all-srcs",
|
||||||
|
srcs = [":package-srcs"],
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
136
pkg/controller/volume/attachdetach/metrics/metrics.go
Normal file
136
pkg/controller/volume/attachdetach/metrics/metrics.go
Normal file
@ -0,0 +1,136 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2018 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/golang/glog"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
corelisters "k8s.io/client-go/listers/core/v1"
|
||||||
|
"k8s.io/kubernetes/pkg/controller/volume/attachdetach/util"
|
||||||
|
"k8s.io/kubernetes/pkg/volume"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
inUseVolumeMetricDesc = prometheus.NewDesc(
|
||||||
|
prometheus.BuildFQName("", "storage_count", "attachable_volumes_in_use"),
|
||||||
|
"Measure number of volumes in use",
|
||||||
|
[]string{"node", "volume_plugin"}, nil)
|
||||||
|
)
|
||||||
|
var registerMetrics sync.Once
|
||||||
|
|
||||||
|
type volumeInUseCollector struct {
|
||||||
|
pvcLister corelisters.PersistentVolumeClaimLister
|
||||||
|
podLister corelisters.PodLister
|
||||||
|
pvLister corelisters.PersistentVolumeLister
|
||||||
|
volumePluginMgr *volume.VolumePluginMgr
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeVolumeCount contains map of {"nodeName": {"pluginName": volume_count }}
|
||||||
|
// For example :
|
||||||
|
// node 172.168.1.100.ec2.internal has 10 EBS and 3 glusterfs PVC in use
|
||||||
|
// {"172.168.1.100.ec2.internal": {"aws-ebs": 10, "glusterfs": 3}}
|
||||||
|
type nodeVolumeCount map[types.NodeName]map[string]int
|
||||||
|
|
||||||
|
// Register registers pvc's in-use metrics
|
||||||
|
func Register(pvcLister corelisters.PersistentVolumeClaimLister,
|
||||||
|
pvLister corelisters.PersistentVolumeLister,
|
||||||
|
podLister corelisters.PodLister,
|
||||||
|
pluginMgr *volume.VolumePluginMgr) {
|
||||||
|
registerMetrics.Do(func() {
|
||||||
|
prometheus.MustRegister(newVolumeInUseCollector(pvcLister, podLister, pvLister, pluginMgr))
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (volumeInUse nodeVolumeCount) add(nodeName types.NodeName, pluginName string) {
|
||||||
|
nodeCount, ok := volumeInUse[nodeName]
|
||||||
|
if !ok {
|
||||||
|
nodeCount = map[string]int{}
|
||||||
|
}
|
||||||
|
nodeCount[pluginName]++
|
||||||
|
volumeInUse[nodeName] = nodeCount
|
||||||
|
}
|
||||||
|
|
||||||
|
func newVolumeInUseCollector(
|
||||||
|
pvcLister corelisters.PersistentVolumeClaimLister,
|
||||||
|
podLister corelisters.PodLister,
|
||||||
|
pvLister corelisters.PersistentVolumeLister,
|
||||||
|
pluginMgr *volume.VolumePluginMgr) *volumeInUseCollector {
|
||||||
|
fmt.Println("Doing this crap again")
|
||||||
|
return &volumeInUseCollector{pvcLister, podLister, pvLister, pluginMgr}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if our collector implements necessary collector interface
|
||||||
|
var _ prometheus.Collector = &volumeInUseCollector{}
|
||||||
|
|
||||||
|
func (collector *volumeInUseCollector) Describe(ch chan<- *prometheus.Desc) {
|
||||||
|
ch <- inUseVolumeMetricDesc
|
||||||
|
}
|
||||||
|
|
||||||
|
func (collector *volumeInUseCollector) Collect(ch chan<- prometheus.Metric) {
|
||||||
|
nodeVolumeMap := collector.getVolumeInUseCount()
|
||||||
|
for nodeName, pluginCount := range nodeVolumeMap {
|
||||||
|
for pluginName, count := range pluginCount {
|
||||||
|
metric, err := prometheus.NewConstMetric(inUseVolumeMetricDesc,
|
||||||
|
prometheus.GaugeValue,
|
||||||
|
float64(count),
|
||||||
|
string(nodeName),
|
||||||
|
pluginName)
|
||||||
|
if err != nil {
|
||||||
|
glog.Warningf("Failed to create metric : %v", err)
|
||||||
|
}
|
||||||
|
ch <- metric
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (collector *volumeInUseCollector) getVolumeInUseCount() nodeVolumeCount {
|
||||||
|
pods, err := collector.podLister.List(labels.Everything())
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("Error getting pod list")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
nodeVolumeMap := make(nodeVolumeCount)
|
||||||
|
for _, pod := range pods {
|
||||||
|
if len(pod.Spec.Volumes) <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
nodeName := types.NodeName(pod.Spec.NodeName)
|
||||||
|
if nodeName == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, podVolume := range pod.Spec.Volumes {
|
||||||
|
volumeSpec, err := util.CreateVolumeSpec(podVolume, pod.Namespace, collector.pvcLister, collector.pvLister)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
volumePlugin, err := collector.volumePluginMgr.FindPluginBySpec(volumeSpec)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
nodeVolumeMap.add(nodeName, volumePlugin.GetPluginName())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nodeVolumeMap
|
||||||
|
}
|
121
pkg/controller/volume/attachdetach/metrics/metrics_test.go
Normal file
121
pkg/controller/volume/attachdetach/metrics/metrics_test.go
Normal file
@ -0,0 +1,121 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2018 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"k8s.io/api/core/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/client-go/informers"
|
||||||
|
"k8s.io/client-go/kubernetes/fake"
|
||||||
|
"k8s.io/kubernetes/pkg/controller"
|
||||||
|
volumetesting "k8s.io/kubernetes/pkg/volume/testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMetricCollection(t *testing.T) {
|
||||||
|
fakeVolumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t)
|
||||||
|
fakeClient := &fake.Clientset{}
|
||||||
|
|
||||||
|
fakeInformerFactory := informers.NewSharedInformerFactory(fakeClient, controller.NoResyncPeriodFunc())
|
||||||
|
fakePodInformer := fakeInformerFactory.Core().V1().Pods()
|
||||||
|
pod := &v1.Pod{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "metric-test-pod",
|
||||||
|
UID: "metric-test-pod-uid",
|
||||||
|
Namespace: "metric-test",
|
||||||
|
},
|
||||||
|
Spec: v1.PodSpec{
|
||||||
|
NodeName: "metric-test-host",
|
||||||
|
Volumes: []v1.Volume{
|
||||||
|
{
|
||||||
|
Name: "metric-test-volume-name",
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||||
|
ClaimName: "metric-test-pvc",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Status: v1.PodStatus{
|
||||||
|
Phase: v1.PodPhase("Running"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
fakePodInformer.Informer().GetStore().Add(pod)
|
||||||
|
pvcInformer := fakeInformerFactory.Core().V1().PersistentVolumeClaims()
|
||||||
|
pvInformer := fakeInformerFactory.Core().V1().PersistentVolumes()
|
||||||
|
|
||||||
|
pvc := &v1.PersistentVolumeClaim{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "metric-test-pvc",
|
||||||
|
Namespace: "metric-test",
|
||||||
|
UID: "metric-test-pvc-1",
|
||||||
|
},
|
||||||
|
Spec: v1.PersistentVolumeClaimSpec{
|
||||||
|
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany, v1.ReadWriteOnce},
|
||||||
|
Resources: v1.ResourceRequirements{
|
||||||
|
Requests: v1.ResourceList{
|
||||||
|
v1.ResourceName(v1.ResourceStorage): resource.MustParse("2G"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
VolumeName: "test-metric-pv-1",
|
||||||
|
},
|
||||||
|
Status: v1.PersistentVolumeClaimStatus{
|
||||||
|
Phase: v1.ClaimBound,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
pv := &v1.PersistentVolume{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
UID: "test-metric-pv-1",
|
||||||
|
Name: "test-metric-pv-1",
|
||||||
|
},
|
||||||
|
Spec: v1.PersistentVolumeSpec{
|
||||||
|
Capacity: v1.ResourceList{
|
||||||
|
v1.ResourceName(v1.ResourceStorage): resource.MustParse("5G"),
|
||||||
|
},
|
||||||
|
PersistentVolumeSource: v1.PersistentVolumeSource{
|
||||||
|
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
|
||||||
|
},
|
||||||
|
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany},
|
||||||
|
// this one we're pretending is already bound
|
||||||
|
ClaimRef: &v1.ObjectReference{UID: "metric-test-pvc-1", Namespace: "metric-test"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
pvcInformer.Informer().GetStore().Add(pvc)
|
||||||
|
pvInformer.Informer().GetStore().Add(pv)
|
||||||
|
pvcLister := pvcInformer.Lister()
|
||||||
|
pvLister := pvInformer.Lister()
|
||||||
|
|
||||||
|
metricCollector := newVolumeInUseCollector(pvcLister, fakePodInformer.Lister(), pvLister, fakeVolumePluginMgr)
|
||||||
|
nodeUseMap := metricCollector.getVolumeInUseCount()
|
||||||
|
if len(nodeUseMap) < 1 {
|
||||||
|
t.Errorf("Expected one volume in use got %d", len(nodeUseMap))
|
||||||
|
}
|
||||||
|
testNodeMetric := nodeUseMap["metric-test-host"]
|
||||||
|
pluginUseCount, ok := testNodeMetric["fake-plugin"]
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("Expected fake plugin pvc got nothing")
|
||||||
|
}
|
||||||
|
|
||||||
|
if pluginUseCount < 1 {
|
||||||
|
t.Errorf("Expected at least in-use volume metric got %d", pluginUseCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user