From 1d144f518e22b6db4fa711d0856154e6db428041 Mon Sep 17 00:00:00 2001
From: Jan Safranek <jsafrane@redhat.com>
Date: Thu, 4 Jan 2024 10:25:10 +0100
Subject: [PATCH] Fix SELinux e2e tests: add label to observed metrics

Some of SELinux relabeling metrics got a new label with volume plugin in
1.29. Add the label to metrics scraping in the SELinux e2e tests.

I had to remove check that all metrics were collected, because metrics with
volume plugin label will start to exist only after an event that raises the
metric happens. They're missing in the initial metric grab.
---
 .../e2e/storage/csi_mock/csi_selinux_mount.go | 66 +++++++++++--------
 1 file changed, 39 insertions(+), 27 deletions(-)

diff --git a/test/e2e/storage/csi_mock/csi_selinux_mount.go b/test/e2e/storage/csi_mock/csi_selinux_mount.go
index cd5cebaa7ab..5c97ec03646 100644
--- a/test/e2e/storage/csi_mock/csi_selinux_mount.go
+++ b/test/e2e/storage/csi_mock/csi_selinux_mount.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -252,19 +253,6 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() {
 
 	// [Serial]: the tests read global kube-controller-manager metrics, so no other test changes them in parallel.
 	f.Context("SELinuxMount metrics [LinuxOnly]", feature.SELinux, feature.SELinuxMountReadWriteOncePod, f.WithSerial(), func() {
-
-		// All SELinux metrics. Unless explicitly mentioned in test.expectIncreases, these metrics must not grow during
-		// a test.
-		allMetrics := sets.NewString(
-			"volume_manager_selinux_container_errors_total",
-			"volume_manager_selinux_container_warnings_total",
-			"volume_manager_selinux_pod_context_mismatch_errors_total",
-			"volume_manager_selinux_pod_context_mismatch_warnings_total",
-			"volume_manager_selinux_volume_context_mismatch_errors_total",
-			"volume_manager_selinux_volume_context_mismatch_warnings_total",
-			"volume_manager_selinux_volumes_admitted_total",
-		)
-
 		// Make sure all options are set so system specific defaults are not used.
 		seLinuxOpts1 := v1.SELinuxOptions{
 			User:  "system_u",
@@ -321,6 +309,21 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() {
 		for _, t := range tests {
 			t := t
 			ginkgo.It(t.name, func(ctx context.Context) {
+				// Some metrics use CSI driver name as a label, which is "csi-mock-" + the namespace name.
+				volumePluginLabel := "{volume_plugin=\"kubernetes.io/csi/csi-mock-" + f.Namespace.Name + "\"}"
+
+				// All SELinux metrics. Unless explicitly mentioned in test.expectIncreases, these metrics must not grow during
+				// a test.
+				allMetrics := sets.NewString(
+					"volume_manager_selinux_container_errors_total",
+					"volume_manager_selinux_container_warnings_total",
+					"volume_manager_selinux_pod_context_mismatch_errors_total",
+					"volume_manager_selinux_pod_context_mismatch_warnings_total",
+					"volume_manager_selinux_volume_context_mismatch_errors_total"+volumePluginLabel,
+					"volume_manager_selinux_volume_context_mismatch_warnings_total"+volumePluginLabel,
+					"volume_manager_selinux_volumes_admitted_total"+volumePluginLabel,
+				)
+
 				if framework.NodeOSDistroIs("windows") {
 					e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping")
 				}
@@ -387,20 +390,17 @@ func grabMetrics(ctx context.Context, grabber *e2emetrics.Grabber, nodeName stri
 	framework.ExpectNoError(err)
 
 	metrics := map[string]float64{}
-	for method, samples := range response {
-		if metricNames.Has(method) {
-			if len(samples) == 0 {
-				return nil, fmt.Errorf("metric %s has no samples", method)
-			}
-			lastSample := samples[len(samples)-1]
-			metrics[method] = float64(lastSample.Value)
+	for _, samples := range response {
+		if len(samples) == 0 {
+			continue
 		}
-	}
-
-	// Ensure all metrics were provided
-	for name := range metricNames {
-		if _, found := metrics[name]; !found {
-			return nil, fmt.Errorf("metric %s not found", name)
+		// Find the *last* sample that has the label we are interested in.
+		for i := len(samples) - 1; i >= 0; i-- {
+			metricNameWithLabels := samples[i].Metric.String()
+			if metricNames.Has(metricNameWithLabels) {
+				metrics[metricNameWithLabels] = float64(samples[i].Value)
+				break
+			}
 		}
 	}
 
@@ -421,11 +421,23 @@ func waitForMetricIncrease(ctx context.Context, grabber *e2emetrics.Grabber, nod
 		noIncreaseMetrics = sets.NewString()
 		// Always evaluate all SELinux metrics to check that the other metrics are not unexpectedly increased.
 		for name := range allMetricNames {
-			if expectedIncreaseNames.Has(name) {
+			expectIncrease := false
+
+			// allMetricNames can include "{volume_plugin="XXX"}", while expectedIncreaseNames does not.
+			// Compare them properly. Value of volume_plugin="XXX" was already checked in grabMetrics(),
+			// we can ignore it here.
+			for expectedIncreaseMetricName := range expectedIncreaseNames {
+				if strings.HasPrefix(name, expectedIncreaseMetricName) {
+					expectIncrease = true
+					break
+				}
+			}
+			if expectIncrease {
 				if metrics[name] <= initialValues[name] {
 					noIncreaseMetrics.Insert(name)
 				}
 			} else {
+				// Expect the metric to be stable
 				if initialValues[name] != metrics[name] {
 					return false, fmt.Errorf("metric %s unexpectedly increased to %v", name, metrics[name])
 				}