From 1d6a701e769d7a49fde3632b8f7be5addcb1aada Mon Sep 17 00:00:00 2001 From: Vaibhav Kamra Date: Thu, 21 Sep 2017 01:10:22 -0700 Subject: [PATCH 1/3] Add e2e test to verify PVC metrics Adds an e2e test for kubernetes/features#363 --- test/e2e/storage/BUILD | 1 + test/e2e/storage/volume_metrics.go | 73 +++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index 409804d7181..8d148acc95c 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -37,6 +37,7 @@ go_library( "//pkg/cloudprovider/providers/vsphere:go_default_library", "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library", "//pkg/kubelet/apis:go_default_library", + "//pkg/kubelet/metrics:go_default_library", "//pkg/volume/util/volumehelper:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/metrics:go_default_library", diff --git a/test/e2e/storage/volume_metrics.go b/test/e2e/storage/volume_metrics.go index 510c5e1347f..969f666ecbe 100644 --- a/test/e2e/storage/volume_metrics.go +++ b/test/e2e/storage/volume_metrics.go @@ -17,11 +17,16 @@ limitations under the License. package storage import ( + "fmt" + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" + kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/metrics" ) @@ -51,7 +56,7 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() { pvc = newClaim(test, ns, "default") var err error - metricsGrabber, err = metrics.NewMetricsGrabber(c, nil, false, false, true, false, false) + metricsGrabber, err = metrics.NewMetricsGrabber(c, nil, true, false, true, false, false) if err != nil { framework.Failf("Error creating metrics grabber : %v", err) @@ -95,6 +100,53 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() { verifyMetricCount(storageOpMetrics, updatedStorageMetrics, volumeOp) } }) + + It("should create prometheus metrics for volume summary stats", func() { + var err error + pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) + Expect(err).NotTo(HaveOccurred()) + Expect(pvc).ToNot(Equal(nil)) + defer func() { + framework.Logf("Deleting claim %q/%q", pvc.Namespace, pvc.Name) + framework.ExpectNoError(c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil)) + }() + + claims := []*v1.PersistentVolumeClaim{pvc} + pod := framework.MakePod(ns, claims, false, "") + pod, err = c.CoreV1().Pods(ns).Create(pod) + Expect(err).NotTo(HaveOccurred()) + + err = framework.WaitForPodRunningInNamespace(c, pod) + framework.ExpectNoError(framework.WaitForPodRunningInNamespace(c, pod), "Error starting pod ", pod.Name) + + pod, err = c.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Wait for `VolumeStatsAggPeriod' to grab metrics + time.Sleep(1 * time.Minute) + + // Grab kubelet metrics from the node the pod was scheduled on + kubeMetrics, err := metricsGrabber.GrabFromKubelet(pod.Spec.NodeName) + Expect(err).NotTo(HaveOccurred(), "Error getting kubelet metrics : %v", err) + + framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name) + framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod)) + + // Verify volume stat metrics were collected for the referenced PVC + volumeStatKeys := []string{ + kubeletmetrics.VolumeStatsUsedBytesKey, + kubeletmetrics.VolumeStatsCapacityBytesKey, + kubeletmetrics.VolumeStatsAvailableBytesKey, + kubeletmetrics.VolumeStatsUsedBytesKey, + kubeletmetrics.VolumeStatsInodesFreeKey, + kubeletmetrics.VolumeStatsInodesUsedKey, + } + + for _, key := range volumeStatKeys { + kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key) + verifyVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics) + } + }) }) func verifyMetricCount(oldMetrics map[string]int64, newMetrics map[string]int64, metricName string) { @@ -123,3 +175,22 @@ func getControllerStorageMetrics(ms metrics.ControllerManagerMetrics) map[string } return result } + +// Verifies the specified metrics are in `kubeletMetrics` +func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName string, kubeletMetrics metrics.KubeletMetrics) { + found := false + if samples, ok := kubeletMetrics[metricKeyName]; ok { + for _, sample := range samples { + samplePVC, ok := sample.Metric["persistentvolumeclaim"] + Expect(ok).To(BeTrue(), "Error getting pvc for %s", metricKeyName) + sampleNS, ok := sample.Metric["namespace"] + Expect(ok).To(BeTrue(), "Error getting namespace for %s", metricKeyName) + + if string(samplePVC) == pvcName && string(sampleNS) == namespace { + found = true + break + } + } + } + Expect(found).To(BeTrue(), "PVC %s, Namespace %s not found for %s", pvcName, namespace, metricKeyName) +} From 41dabd791845b4d2e2c79ac4af36c2fb87c5734d Mon Sep 17 00:00:00 2001 From: Vaibhav Kamra Date: Thu, 21 Sep 2017 13:44:06 -0700 Subject: [PATCH 2/3] Address review comments --- test/e2e/storage/volume_metrics.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/test/e2e/storage/volume_metrics.go b/test/e2e/storage/volume_metrics.go index 969f666ecbe..da2db3786ee 100644 --- a/test/e2e/storage/volume_metrics.go +++ b/test/e2e/storage/volume_metrics.go @@ -63,6 +63,12 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() { } }) + AfterEach(func() { + defer func() { + framework.DeletePersistentVolumeClaim(c, pvc.Name, pvc.Namespace) + }() + }) + It("should create prometheus metrics for volume provisioning and attach/detach", func() { var err error @@ -74,10 +80,6 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() { pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) Expect(err).NotTo(HaveOccurred()) Expect(pvc).ToNot(Equal(nil)) - defer func() { - framework.Logf("Deleting claim %q/%q", pvc.Namespace, pvc.Name) - framework.ExpectNoError(c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil)) - }() claims := []*v1.PersistentVolumeClaim{pvc} @@ -101,15 +103,11 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() { } }) - It("should create prometheus metrics for volume summary stats", func() { + It("should create volume metrics with the correct PVC ref", func() { var err error pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) Expect(err).NotTo(HaveOccurred()) Expect(pvc).ToNot(Equal(nil)) - defer func() { - framework.Logf("Deleting claim %q/%q", pvc.Namespace, pvc.Name) - framework.ExpectNoError(c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil)) - }() claims := []*v1.PersistentVolumeClaim{pvc} pod := framework.MakePod(ns, claims, false, "") @@ -179,12 +177,19 @@ func getControllerStorageMetrics(ms metrics.ControllerManagerMetrics) map[string // Verifies the specified metrics are in `kubeletMetrics` func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName string, kubeletMetrics metrics.KubeletMetrics) { found := false + invalidSamples := []string{} if samples, ok := kubeletMetrics[metricKeyName]; ok { for _, sample := range samples { samplePVC, ok := sample.Metric["persistentvolumeclaim"] - Expect(ok).To(BeTrue(), "Error getting pvc for %s", metricKeyName) + if !ok { + framework.Logf("Error getting pvc for metric %s, sample %s", metricKeyName, sample.String()) + invalidSamples = append(invalidSamples, sample.String()) + } sampleNS, ok := sample.Metric["namespace"] - Expect(ok).To(BeTrue(), "Error getting namespace for %s", metricKeyName) + if !ok { + framework.Logf("Error getting namespace for metric %s, sample %s", metricKeyName, sample.String()) + invalidSamples = append(invalidSamples, sample.String()) + } if string(samplePVC) == pvcName && string(sampleNS) == namespace { found = true @@ -192,5 +197,6 @@ func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName stri } } } + Expect(invalidSamples).To(HaveLen(0), "Found %d invalid samples", len(invalidSamples)) Expect(found).To(BeTrue(), "PVC %s, Namespace %s not found for %s", pvcName, namespace, metricKeyName) } From b3ced98aa5ae15b3fad87cbc9998be7690f5ba74 Mon Sep 17 00:00:00 2001 From: Vaibhav Kamra Date: Thu, 21 Sep 2017 14:24:26 -0700 Subject: [PATCH 3/3] Address review comment Removed defer, removed error slice --- test/e2e/storage/volume_metrics.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/e2e/storage/volume_metrics.go b/test/e2e/storage/volume_metrics.go index da2db3786ee..93f2a6f7893 100644 --- a/test/e2e/storage/volume_metrics.go +++ b/test/e2e/storage/volume_metrics.go @@ -64,9 +64,7 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() { }) AfterEach(func() { - defer func() { - framework.DeletePersistentVolumeClaim(c, pvc.Name, pvc.Namespace) - }() + framework.DeletePersistentVolumeClaim(c, pvc.Name, pvc.Namespace) }) It("should create prometheus metrics for volume provisioning and attach/detach", func() { @@ -177,18 +175,18 @@ func getControllerStorageMetrics(ms metrics.ControllerManagerMetrics) map[string // Verifies the specified metrics are in `kubeletMetrics` func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName string, kubeletMetrics metrics.KubeletMetrics) { found := false - invalidSamples := []string{} + errCount := 0 if samples, ok := kubeletMetrics[metricKeyName]; ok { for _, sample := range samples { samplePVC, ok := sample.Metric["persistentvolumeclaim"] if !ok { framework.Logf("Error getting pvc for metric %s, sample %s", metricKeyName, sample.String()) - invalidSamples = append(invalidSamples, sample.String()) + errCount++ } sampleNS, ok := sample.Metric["namespace"] if !ok { framework.Logf("Error getting namespace for metric %s, sample %s", metricKeyName, sample.String()) - invalidSamples = append(invalidSamples, sample.String()) + errCount++ } if string(samplePVC) == pvcName && string(sampleNS) == namespace { @@ -197,6 +195,6 @@ func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName stri } } } - Expect(invalidSamples).To(HaveLen(0), "Found %d invalid samples", len(invalidSamples)) + Expect(errCount).To(Equal(0), "Found invalid samples") Expect(found).To(BeTrue(), "PVC %s, Namespace %s not found for %s", pvcName, namespace, metricKeyName) }