github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 20:53:33 +00:00
Code Issues Projects Releases Wiki Activity

storage e2e: fix volume metric test for generic ephemeral volume

Browse Source 
This is a fix for the new test case from
https://github.com/kubernetes/kubernetes/pull/105636 which had to be merged
without prior testing due to not having a cluster to test on and no pull job
which runs these
tests. https://testgrid.k8s.io/sig-storage-kubernetes#gce-serial then showed a
failure.

The fix is simple: in the ephemeral case, the PVC name isn't set in advance in
pvc.Name and instead must be computed. The fix now was tested on a kubetest
cluster in GCE.

Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
This commit is contained in:
Patrick Ohly 2021-10-14 16:28:26 +02:00
parent 4568cdada2
commit 7538d089d5
1 changed files with 36 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
test/e2e/storage/volume_metrics.go
View File

@ -30,6 +30,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
"k8s.io/component-base/metrics/testutil" "k8s.io/component-base/metrics/testutil"
"k8s.io/component-helpers/storage/ephemeral"
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
@ -211,14 +212,15 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
verifyMetricCount(storageOpMetrics, updatedStorageMetrics, "volume_provision", true) verifyMetricCount(storageOpMetrics, updatedStorageMetrics, "volume_provision", true)
} }
filesystemMode := func(ephemeral bool) { filesystemMode := func(isEphemeral bool) {
if !ephemeral { pvcName := pvc.Name
if !isEphemeral {
pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}) pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
framework.ExpectNotEqual(pvc, nil) framework.ExpectNotEqual(pvc, nil)
} }
pod := makePod(ns, pvc, ephemeral) pod := makePod(ns, pvc, isEphemeral)
pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
@ -228,6 +230,11 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
pod, err = c.CoreV1().Pods(ns).Get(context.TODO(), pod.Name, metav1.GetOptions{}) pod, err = c.CoreV1().Pods(ns).Get(context.TODO(), pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
if isEphemeral {
pvcName = ephemeral.VolumeClaimName(pod, &pod.Spec.Volumes[0])
}
pvcNamespace := pod.Namespace
// Verify volume stat metrics were collected for the referenced PVC // Verify volume stat metrics were collected for the referenced PVC
volumeStatKeys := []string{ volumeStatKeys := []string{
kubeletmetrics.VolumeStatsUsedBytesKey, kubeletmetrics.VolumeStatsUsedBytesKey,
@ -251,31 +258,31 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
framework.Logf("Error fetching kubelet metrics") framework.Logf("Error fetching kubelet metrics")
return false, err return false, err
} }
if !findVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics) { if !findVolumeStatMetric(kubeletKeyName, pvcNamespace, pvcName, kubeMetrics) {
return false, nil return false, nil
} }
return true, nil return true, nil
}) })
framework.ExpectNoError(waitErr, "Unable to find metric %s for PVC %s/%s", kubeletKeyName, pvc.Namespace, pvc.Name) framework.ExpectNoError(waitErr, "Unable to find metric %s for PVC %s/%s", kubeletKeyName, pvcNamespace, pvcName)
for _, key := range volumeStatKeys { for _, key := range volumeStatKeys {
kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key) kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key)
found := findVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics) found := findVolumeStatMetric(kubeletKeyName, pvcNamespace, pvcName, kubeMetrics)
framework.ExpectEqual(found, true, "PVC %s, Namespace %s not found for %s", pvc.Name, pvc.Namespace, kubeletKeyName) framework.ExpectEqual(found, true, "PVC %s, Namespace %s not found for %s", pvcName, pvcNamespace, kubeletKeyName)
} }
framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name) framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod)) framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
} }
blockmode := func(ephemeral bool) { blockmode := func(isEphemeral bool) {
if !ephemeral { if !isEphemeral {
pvcBlock, err = c.CoreV1().PersistentVolumeClaims(pvcBlock.Namespace).Create(context.TODO(), pvcBlock, metav1.CreateOptions{}) pvcBlock, err = c.CoreV1().PersistentVolumeClaims(pvcBlock.Namespace).Create(context.TODO(), pvcBlock, metav1.CreateOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
framework.ExpectNotEqual(pvcBlock, nil) framework.ExpectNotEqual(pvcBlock, nil)
} }
pod := makePod(ns, pvcBlock, ephemeral) pod := makePod(ns, pvcBlock, isEphemeral)
pod.Spec.Containers[0].VolumeDevices = []v1.VolumeDevice{{ pod.Spec.Containers[0].VolumeDevices = []v1.VolumeDevice{{
Name: pod.Spec.Volumes[0].Name, Name: pod.Spec.Volumes[0].Name,
DevicePath: "/mnt/" + pvcBlock.Name, DevicePath: "/mnt/" + pvcBlock.Name,
@ -326,14 +333,14 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod)) framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
} }
totalTime := func(ephemeral bool) { totalTime := func(isEphemeral bool) {
if !ephemeral { if !isEphemeral {
pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}) pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
framework.ExpectNotEqual(pvc, nil) framework.ExpectNotEqual(pvc, nil)
} }
pod := makePod(ns, pvc, ephemeral) pod := makePod(ns, pvc, isEphemeral)
pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
@ -357,14 +364,14 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod)) framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
} }
volumeManager := func(ephemeral bool) { volumeManager := func(isEphemeral bool) {
if !ephemeral { if !isEphemeral {
pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}) pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
framework.ExpectNotEqual(pvc, nil) framework.ExpectNotEqual(pvc, nil)
} }
pod := makePod(ns, pvc, ephemeral) pod := makePod(ns, pvc, isEphemeral)
pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) pod, err = c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
@ -387,14 +394,14 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod)) framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
} }
adController := func(ephemeral bool) { adController := func(isEphemeral bool) {
if !ephemeral { if !isEphemeral {
pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}) pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
framework.ExpectNoError(err) framework.ExpectNoError(err)
framework.ExpectNotEqual(pvc, nil) framework.ExpectNotEqual(pvc, nil)
} }
pod := makePod(ns, pvc, ephemeral) pod := makePod(ns, pvc, isEphemeral)
// Get metrics // Get metrics
controllerMetrics, err := metricsGrabber.GrabFromControllerManager() controllerMetrics, err := metricsGrabber.GrabFromControllerManager()
@ -447,27 +454,27 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod)) framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod))
} }
testAll := func(ephemeral bool) { testAll := func(isEphemeral bool) {
ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() { ginkgo.It("should create prometheus metrics for volume provisioning and attach/detach", func() {
provisioning(ephemeral) provisioning(isEphemeral)
}) })
ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() { ginkgo.It("should create prometheus metrics for volume provisioning errors [Slow]", func() {
provisioningError(ephemeral) provisioningError(isEphemeral)
}) })
ginkgo.It("should create volume metrics with the correct FilesystemMode PVC ref", func() { ginkgo.It("should create volume metrics with the correct FilesystemMode PVC ref", func() {
filesystemMode(ephemeral) filesystemMode(isEphemeral)
}) })
ginkgo.It("should create volume metrics with the correct BlockMode PVC ref", func() { ginkgo.It("should create volume metrics with the correct BlockMode PVC ref", func() {
blockmode(ephemeral) blockmode(isEphemeral)
}) })
ginkgo.It("should create metrics for total time taken in volume operations in P/V Controller", func() { ginkgo.It("should create metrics for total time taken in volume operations in P/V Controller", func() {
totalTime(ephemeral) totalTime(isEphemeral)
}) })
ginkgo.It("should create volume metrics in Volume Manager", func() { ginkgo.It("should create volume metrics in Volume Manager", func() {
volumeManager(ephemeral) volumeManager(isEphemeral)
}) })
ginkgo.It("should create metrics for total number of volumes in A/D Controller", func() { ginkgo.It("should create metrics for total number of volumes in A/D Controller", func() {
adController(ephemeral) adController(isEphemeral)
}) })
} }
@ -873,10 +880,10 @@ func waitForADControllerStatesMetrics(metricsGrabber *e2emetrics.Grabber, metric
// makePod creates a pod which either references the PVC or creates it via a // makePod creates a pod which either references the PVC or creates it via a
// generic ephemeral volume claim template. // generic ephemeral volume claim template.
func makePod(ns string, pvc *v1.PersistentVolumeClaim, ephemeral bool) *v1.Pod { func makePod(ns string, pvc *v1.PersistentVolumeClaim, isEphemeral bool) *v1.Pod {
claims := []*v1.PersistentVolumeClaim{pvc} claims := []*v1.PersistentVolumeClaim{pvc}
pod := e2epod.MakePod(ns, nil, claims, false, "") pod := e2epod.MakePod(ns, nil, claims, false, "")
if ephemeral { if isEphemeral {
volSrc := pod.Spec.Volumes[0] volSrc := pod.Spec.Volumes[0]
volSrc.PersistentVolumeClaim = nil volSrc.PersistentVolumeClaim = nil
volSrc.Ephemeral = &v1.EphemeralVolumeSource{ volSrc.Ephemeral = &v1.EphemeralVolumeSource{