github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-30 23:57:46 +00:00
Code Issues Projects Releases Wiki Activity

Improve PVC ref volume metric test robustness

Browse Source 
This test has been flaking. The current working theory is that
volume stats collection didn't run in time to grab the metrics
from the newly created pod.

Made the following changes:
 - Added more logs to help debug future failures
 - Poll metrics a few additional times before failing the test
This commit is contained in:
Vaibhav Kamra 2017-09-27 17:49:05 -07:00
parent acbfde3914
commit c0d9bdaf5e
1 changed files with 26 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
test/e2e/storage/volume_metrics.go
View File

@ -145,16 +145,6 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() {
pod, err = c.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// Wait for `VolumeStatsAggPeriod' to grab metrics
time.Sleep(1 * time.Minute)
// Grab kubelet metrics from the node the pod was scheduled on
kubeMetrics, err := metricsGrabber.GrabFromKubelet(pod.Spec.NodeName)
Expect(err).NotTo(HaveOccurred(), "Error getting kubelet metrics : %v", err)
framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod))
// Verify volume stat metrics were collected for the referenced PVC
volumeStatKeys := []string{
kubeletmetrics.VolumeStatsUsedBytesKey,
@ -165,10 +155,26 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() {
kubeletmetrics.VolumeStatsInodesUsedKey,
}
for _, key := range volumeStatKeys {
kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key)
verifyVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics)
}
waitErr := wait.Poll(30*time.Second, 5*time.Minute, func() (bool, error) {
// Grab kubelet metrics from the node the pod was scheduled on
kubeMetrics, err := metricsGrabber.GrabFromKubelet(pod.Spec.NodeName)
if err != nil {
framework.Logf("Error fetching kubelet metrics")
return false, err
}
for _, key := range volumeStatKeys {
kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key)
if !findVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics) {
return false, nil
}
}
return true, nil
})
Expect(waitErr).NotTo(HaveOccurred(), "Error finding volume metrics : %v", waitErr)
framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod))
})
})
@ -202,12 +208,15 @@ func getControllerStorageMetrics(ms metrics.ControllerManagerMetrics) map[string
return result
}
// Verifies the specified metrics are in `kubeletMetrics`
func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName string, kubeletMetrics metrics.KubeletMetrics) {
// Finds the sample in the specified metric from `KubeletMetrics` tagged with
// the specified namespace and pvc name
func findVolumeStatMetric(metricKeyName string, namespace string, pvcName string, kubeletMetrics metrics.KubeletMetrics) bool {
found := false
errCount := 0
framework.Logf("Looking for sample in metric `%s` tagged with namespace `%s`, PVC `%s`", metricKeyName, namespace, pvcName)
if samples, ok := kubeletMetrics[metricKeyName]; ok {
for _, sample := range samples {
framework.Logf("Found sample %s", sample.String())
samplePVC, ok := sample.Metric["persistentvolumeclaim"]
if !ok {
framework.Logf("Error getting pvc for metric %s, sample %s", metricKeyName, sample.String())
@ -226,5 +235,5 @@ func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName stri
}
}
Expect(errCount).To(Equal(0), "Found invalid samples")
Expect(found).To(BeTrue(), "PVC %s, Namespace %s not found for %s", pvcName, namespace, metricKeyName)
return found
}