diff --git a/pkg/controller/volume/attachdetach/BUILD b/pkg/controller/volume/attachdetach/BUILD index 156b8180184..13dbe301cab 100644 --- a/pkg/controller/volume/attachdetach/BUILD +++ b/pkg/controller/volume/attachdetach/BUILD @@ -14,6 +14,7 @@ go_library( "//pkg/cloudprovider:go_default_library", "//pkg/controller:go_default_library", "//pkg/controller/volume/attachdetach/cache:go_default_library", + "//pkg/controller/volume/attachdetach/metrics:go_default_library", "//pkg/controller/volume/attachdetach/populator:go_default_library", "//pkg/controller/volume/attachdetach/reconciler:go_default_library", "//pkg/controller/volume/attachdetach/statusupdater:go_default_library", @@ -69,6 +70,7 @@ filegroup( srcs = [ ":package-srcs", "//pkg/controller/volume/attachdetach/cache:all-srcs", + "//pkg/controller/volume/attachdetach/metrics:all-srcs", "//pkg/controller/volume/attachdetach/populator:all-srcs", "//pkg/controller/volume/attachdetach/reconciler:all-srcs", "//pkg/controller/volume/attachdetach/statusupdater:all-srcs", diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index 805a304952f..1847d2a2f39 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -39,6 +39,7 @@ import ( "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" + "k8s.io/kubernetes/pkg/controller/volume/attachdetach/metrics" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/populator" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater" @@ -273,6 +274,7 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) { } go adc.reconciler.Run(stopCh) go adc.desiredStateOfWorldPopulator.Run(stopCh) + metrics.Register(adc.pvcLister, adc.pvLister, adc.podLister, &adc.volumePluginMgr) <-stopCh } diff --git a/pkg/controller/volume/attachdetach/metrics/BUILD b/pkg/controller/volume/attachdetach/metrics/BUILD new file mode 100644 index 00000000000..4a57a114119 --- /dev/null +++ b/pkg/controller/volume/attachdetach/metrics/BUILD @@ -0,0 +1,46 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["metrics.go"], + importpath = "k8s.io/kubernetes/pkg/controller/volume/attachdetach/metrics", + visibility = ["//visibility:public"], + deps = [ + "//pkg/controller/volume/attachdetach/util:go_default_library", + "//pkg/volume:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["metrics_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/controller:go_default_library", + "//pkg/volume/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/controller/volume/attachdetach/metrics/metrics.go b/pkg/controller/volume/attachdetach/metrics/metrics.go new file mode 100644 index 00000000000..d42f63e8d1f --- /dev/null +++ b/pkg/controller/volume/attachdetach/metrics/metrics.go @@ -0,0 +1,136 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "fmt" + "sync" + + "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/kubernetes/pkg/controller/volume/attachdetach/util" + "k8s.io/kubernetes/pkg/volume" +) + +var ( + inUseVolumeMetricDesc = prometheus.NewDesc( + prometheus.BuildFQName("", "storage_count", "attachable_volumes_in_use"), + "Measure number of volumes in use", + []string{"node", "volume_plugin"}, nil) +) +var registerMetrics sync.Once + +type volumeInUseCollector struct { + pvcLister corelisters.PersistentVolumeClaimLister + podLister corelisters.PodLister + pvLister corelisters.PersistentVolumeLister + volumePluginMgr *volume.VolumePluginMgr +} + +// nodeVolumeCount contains map of {"nodeName": {"pluginName": volume_count }} +// For example : +// node 172.168.1.100.ec2.internal has 10 EBS and 3 glusterfs PVC in use +// {"172.168.1.100.ec2.internal": {"aws-ebs": 10, "glusterfs": 3}} +type nodeVolumeCount map[types.NodeName]map[string]int + +// Register registers pvc's in-use metrics +func Register(pvcLister corelisters.PersistentVolumeClaimLister, + pvLister corelisters.PersistentVolumeLister, + podLister corelisters.PodLister, + pluginMgr *volume.VolumePluginMgr) { + registerMetrics.Do(func() { + prometheus.MustRegister(newVolumeInUseCollector(pvcLister, podLister, pvLister, pluginMgr)) + }) + +} + +func (volumeInUse nodeVolumeCount) add(nodeName types.NodeName, pluginName string) { + nodeCount, ok := volumeInUse[nodeName] + if !ok { + nodeCount = map[string]int{} + } + nodeCount[pluginName]++ + volumeInUse[nodeName] = nodeCount +} + +func newVolumeInUseCollector( + pvcLister corelisters.PersistentVolumeClaimLister, + podLister corelisters.PodLister, + pvLister corelisters.PersistentVolumeLister, + pluginMgr *volume.VolumePluginMgr) *volumeInUseCollector { + fmt.Println("Doing this crap again") + return &volumeInUseCollector{pvcLister, podLister, pvLister, pluginMgr} +} + +// Check if our collector implements necessary collector interface +var _ prometheus.Collector = &volumeInUseCollector{} + +func (collector *volumeInUseCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- inUseVolumeMetricDesc +} + +func (collector *volumeInUseCollector) Collect(ch chan<- prometheus.Metric) { + nodeVolumeMap := collector.getVolumeInUseCount() + for nodeName, pluginCount := range nodeVolumeMap { + for pluginName, count := range pluginCount { + metric, err := prometheus.NewConstMetric(inUseVolumeMetricDesc, + prometheus.GaugeValue, + float64(count), + string(nodeName), + pluginName) + if err != nil { + glog.Warningf("Failed to create metric : %v", err) + } + ch <- metric + } + } +} + +func (collector *volumeInUseCollector) getVolumeInUseCount() nodeVolumeCount { + pods, err := collector.podLister.List(labels.Everything()) + if err != nil { + glog.Errorf("Error getting pod list") + return nil + } + + nodeVolumeMap := make(nodeVolumeCount) + for _, pod := range pods { + if len(pod.Spec.Volumes) <= 0 { + continue + } + + nodeName := types.NodeName(pod.Spec.NodeName) + if nodeName == "" { + continue + } + for _, podVolume := range pod.Spec.Volumes { + volumeSpec, err := util.CreateVolumeSpec(podVolume, pod.Namespace, collector.pvcLister, collector.pvLister) + if err != nil { + continue + } + volumePlugin, err := collector.volumePluginMgr.FindPluginBySpec(volumeSpec) + if err != nil { + continue + } + nodeVolumeMap.add(nodeName, volumePlugin.GetPluginName()) + } + } + return nodeVolumeMap +} diff --git a/pkg/controller/volume/attachdetach/metrics/metrics_test.go b/pkg/controller/volume/attachdetach/metrics/metrics_test.go new file mode 100644 index 00000000000..4916e2e5dbb --- /dev/null +++ b/pkg/controller/volume/attachdetach/metrics/metrics_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "testing" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/kubernetes/pkg/controller" + volumetesting "k8s.io/kubernetes/pkg/volume/testing" +) + +func TestMetricCollection(t *testing.T) { + fakeVolumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) + fakeClient := &fake.Clientset{} + + fakeInformerFactory := informers.NewSharedInformerFactory(fakeClient, controller.NoResyncPeriodFunc()) + fakePodInformer := fakeInformerFactory.Core().V1().Pods() + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metric-test-pod", + UID: "metric-test-pod-uid", + Namespace: "metric-test", + }, + Spec: v1.PodSpec{ + NodeName: "metric-test-host", + Volumes: []v1.Volume{ + { + Name: "metric-test-volume-name", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "metric-test-pvc", + }, + }, + }, + }, + }, + Status: v1.PodStatus{ + Phase: v1.PodPhase("Running"), + }, + } + + fakePodInformer.Informer().GetStore().Add(pod) + pvcInformer := fakeInformerFactory.Core().V1().PersistentVolumeClaims() + pvInformer := fakeInformerFactory.Core().V1().PersistentVolumes() + + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metric-test-pvc", + Namespace: "metric-test", + UID: "metric-test-pvc-1", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany, v1.ReadWriteOnce}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse("2G"), + }, + }, + VolumeName: "test-metric-pv-1", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + } + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + UID: "test-metric-pv-1", + Name: "test-metric-pv-1", + }, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse("5G"), + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}, + }, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}, + // this one we're pretending is already bound + ClaimRef: &v1.ObjectReference{UID: "metric-test-pvc-1", Namespace: "metric-test"}, + }, + } + pvcInformer.Informer().GetStore().Add(pvc) + pvInformer.Informer().GetStore().Add(pv) + pvcLister := pvcInformer.Lister() + pvLister := pvInformer.Lister() + + metricCollector := newVolumeInUseCollector(pvcLister, fakePodInformer.Lister(), pvLister, fakeVolumePluginMgr) + nodeUseMap := metricCollector.getVolumeInUseCount() + if len(nodeUseMap) < 1 { + t.Errorf("Expected one volume in use got %d", len(nodeUseMap)) + } + testNodeMetric := nodeUseMap["metric-test-host"] + pluginUseCount, ok := testNodeMetric["fake-plugin"] + if !ok { + t.Errorf("Expected fake plugin pvc got nothing") + } + + if pluginUseCount < 1 { + t.Errorf("Expected at least in-use volume metric got %d", pluginUseCount) + } + +}