Add e2e testing for Block volume metrics

The in-tree drivers support gathering the capacity of the Block volume.
Make sure that Kubelet exposes these for the matching PVCs.

test/e2e/storage/volume_metrics.go

@@ -47,6 +47,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		c              clientset.Interface
 		ns             string
 		pvc            *v1.PersistentVolumeClaim
+		pvcBlock       *v1.PersistentVolumeClaim
 		metricsGrabber *e2emetrics.Grabber
 		invalidSc      *storagev1.StorageClass
 		defaultScName  string
@@ -67,9 +68,17 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 			ClaimSize: "2Gi",
 		}
 
+		fsMode := v1.PersistentVolumeFilesystem
 		pvc = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
 			ClaimSize:  test.ClaimSize,
-			VolumeMode: &test.VolumeMode,
+			VolumeMode: &fsMode,
+		}, ns)
+
+		// selected providers all support PersistentVolumeBlock
+		blockMode := v1.PersistentVolumeBlock
+		pvcBlock = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
+			ClaimSize:  test.ClaimSize,
+			VolumeMode: &blockMode,
 		}, ns)
 
 		metricsGrabber, err = e2emetrics.NewMetricsGrabber(c, nil, true, false, true, false, false)
@@ -201,7 +210,7 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		verifyMetricCount(storageOpMetrics, updatedStorageMetrics, "volume_provision", true)
 	})
 
-	ginkgo.It("should create volume metrics with the correct PVC ref", func() {
+	ginkgo.It("should create volume metrics with the correct FilesystemMode PVC ref", func() {
 		var err error
 		pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
 		framework.ExpectNoError(err)
@@ -258,6 +267,71 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
 	})
 
+	ginkgo.It("should create volume metrics with the correct BlockMode PVC ref", func() {
+		var err error
+		pvcBlock, err = c.CoreV1().PersistentVolumeClaims(pvcBlock.Namespace).Create(context.TODO(), pvcBlock, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+		framework.ExpectNotEqual(pvcBlock, nil)
+
+		pod := e2epod.MakePod(ns, nil, nil, false, "")
+		pod.Spec.Containers[0].VolumeDevices = []v1.VolumeDevice{{
+			Name:       pvcBlock.Name,
+			DevicePath: "/mnt/" + pvcBlock.Name,
+		}}
+		pod.Spec.Volumes = []v1.Volume{{
+			Name: pvcBlock.Name,
+			VolumeSource: v1.VolumeSource{
+				PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+					ClaimName: pvcBlock.Name,
+					ReadOnly:  false,
+				},
+			},
+		}}
+		pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
+		err = e2epod.WaitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, f.Timeouts.PodStart)
+		framework.ExpectNoError(err, "Error starting pod ", pod.Name)
+
+		pod, err = c.CoreV1().Pods(ns).Get(context.TODO(), pod.Name, metav1.GetOptions{})
+		framework.ExpectNoError(err)
+
+		// Verify volume stat metrics were collected for the referenced PVC
+		volumeStatKeys := []string{
+			// BlockMode PVCs only support capacity (for now)
+			kubeletmetrics.VolumeStatsCapacityBytesKey,
+		}
+		key := volumeStatKeys[0]
+		kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key)
+		// Poll kubelet metrics waiting for the volume to be picked up
+		// by the volume stats collector
+		var kubeMetrics e2emetrics.KubeletMetrics
+		waitErr := wait.Poll(30*time.Second, 5*time.Minute, func() (bool, error) {
+			framework.Logf("Grabbing Kubelet metrics")
+			// Grab kubelet metrics from the node the pod was scheduled on
+			var err error
+			kubeMetrics, err = metricsGrabber.GrabFromKubelet(pod.Spec.NodeName)
+			if err != nil {
+				framework.Logf("Error fetching kubelet metrics")
+				return false, err
+			}
+			if !findVolumeStatMetric(kubeletKeyName, pvcBlock.Namespace, pvcBlock.Name, kubeMetrics) {
+				return false, nil
+			}
+			return true, nil
+		})
+		framework.ExpectNoError(waitErr, "Unable to find metric %s for PVC %s/%s", kubeletKeyName, pvcBlock.Namespace, pvcBlock.Name)
+
+		for _, key := range volumeStatKeys {
+			kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key)
+			found := findVolumeStatMetric(kubeletKeyName, pvcBlock.Namespace, pvcBlock.Name, kubeMetrics)
+			framework.ExpectEqual(found, true, "PVC %s, Namespace %s not found for %s", pvcBlock.Name, pvcBlock.Namespace, kubeletKeyName)
+		}
+
+		framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
+		framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
+	})
+
 	ginkgo.It("should create metrics for total time taken in volume operations in P/V Controller", func() {
 		var err error
 		pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})