github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-01-29 21:29:24 +00:00
Code Issues Projects Releases Wiki Activity

Add label with access mode to SELinux metrics

Browse Source 
In the KEP 1710 we promised to have all SELinux metrics with access mode
label, so cluster admin is able to distinguish when RWOP volumes are
failing to mount (-> SELinuxMountReadWriteOncePod feature gate must be
disabled) or volumes with any other access modes are failing (->
SELinuxMount feature gate must be disabled).

Adding the label to kubelet is quite straightforward, there were some
changes needed in the e2e test. Now grabMetrics() collects values of all
SELinux related metrics with all labels. It only skips unrelated volume
plugins. And waitForMetricIncrease gets metric with all labels on input, so
it can check that say RWOP metric increased and RWX one did not.
This commit is contained in:
Jan Safranek
2024-02-29 14:19:16 +01:00
parent a4eaf6e120
commit c4163a9cb8
3 changed files with 133 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
test/e2e/storage/csi_mock/csi_selinux_mount.go
View File

@@ -31,6 +31,8 @@ import (
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/component-base/metrics/testutil"
"k8s.io/kubernetes/pkg/apis/core/v1/helper"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/test/e2e/feature"
@@ -316,6 +318,24 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() {
})
})
var (
// SELinux metrics that have volume_plugin and access_mode labels
metricsWithVolumePluginLabel = sets.NewString(
"volume_manager_selinux_volume_context_mismatch_errors_total",
"volume_manager_selinux_volume_context_mismatch_warnings_total",
"volume_manager_selinux_volumes_admitted_total",
)
// SELinuxMetrics that have only access_mode label
metricsWithoutVolumePluginLabel = sets.NewString(
"volume_manager_selinux_container_errors_total",
"volume_manager_selinux_container_warnings_total",
"volume_manager_selinux_pod_context_mismatch_errors_total",
"volume_manager_selinux_pod_context_mismatch_warnings_total",
)
// All SELinux metrics
allMetrics = metricsWithoutVolumePluginLabel.Union(metricsWithVolumePluginLabel)
)
var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() {
f := framework.NewDefaultFramework("csi-mock-volumes-selinux-metrics")
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
@@ -389,6 +409,17 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() {
expectIncreases: sets.NewString("volume_manager_selinux_volume_context_mismatch_errors_total"),
testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)},
},
{
name: "error is bumped on two Pods with a different context on RWX volume and SELinuxMount enabled",
csiDriverSELinuxEnabled: true,
firstPodSELinuxOpts: &seLinuxOpts1,
secondPodSELinuxOpts: &seLinuxOpts2,
secondPodFailureEvent: "conflicting SELinux labels of volume",
volumeMode: v1.ReadWriteMany,
waitForSecondPodStart: false,
expectIncreases: sets.NewString("volume_manager_selinux_volume_context_mismatch_errors_total"),
testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)},
},
{
name: "error is bumped on two Pods with a different context on RWOP volume",
csiDriverSELinuxEnabled: true,
@@ -405,19 +436,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() {
t := t
testFunc := func(ctx context.Context) {
// Some metrics use CSI driver name as a label, which is "csi-mock-" + the namespace name.
volumePluginLabel := "{volume_plugin=\"kubernetes.io/csi/csi-mock-" + f.Namespace.Name + "\"}"
// All SELinux metrics. Unless explicitly mentioned in test.expectIncreases, these metrics must not grow during
// a test.
allMetrics := sets.NewString(
"volume_manager_selinux_container_errors_total",
"volume_manager_selinux_container_warnings_total",
"volume_manager_selinux_pod_context_mismatch_errors_total",
"volume_manager_selinux_pod_context_mismatch_warnings_total",
"volume_manager_selinux_volume_context_mismatch_errors_total"+volumePluginLabel,
"volume_manager_selinux_volume_context_mismatch_warnings_total"+volumePluginLabel,
"volume_manager_selinux_volumes_admitted_total"+volumePluginLabel,
)
volumePluginLabel := "volume_plugin=\"kubernetes.io/csi/csi-mock-" + f.Namespace.Name + "\""
if framework.NodeOSDistroIs("windows") {
e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping")
@@ -444,7 +463,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() {
ginkgo.By("Grabbing initial metrics")
pod, err = m.cs.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "getting the initial pod")
metrics, err := grabMetrics(ctx, grabber, pod.Spec.NodeName, allMetrics)
metrics, err := grabMetrics(ctx, grabber, pod.Spec.NodeName, allMetrics, volumePluginLabel)
framework.ExpectNoError(err, "collecting the initial metrics")
dumpMetrics(metrics)
@@ -472,8 +491,9 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() {
}
// Assert: count the metrics
ginkgo.By("Waiting for expected metric changes")
err = waitForMetricIncrease(ctx, grabber, pod.Spec.NodeName, allMetrics, t.expectIncreases, metrics, framework.PodStartShortTimeout)
expectIncreaseWithLabels := addLabels(t.expectIncreases, volumePluginLabel, t.volumeMode)
framework.Logf("Waiting for changes of metrics %+v", expectIncreaseWithLabels)
err = waitForMetricIncrease(ctx, grabber, pod.Spec.NodeName, volumePluginLabel, allMetrics, expectIncreaseWithLabels, metrics, framework.PodStartShortTimeout)
framework.ExpectNoError(err, "waiting for metrics %s to increase", t.expectIncreases)
}
// t.testTags is array and it's not possible to use It("name", func(){xxx}, t.testTags...)
@@ -488,7 +508,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() {
})
})
func grabMetrics(ctx context.Context, grabber *e2emetrics.Grabber, nodeName string, metricNames sets.String) (map[string]float64, error) {
func grabMetrics(ctx context.Context, grabber *e2emetrics.Grabber, nodeName string, metricNames sets.String, volumePluginLabel string) (map[string]float64, error) {
response, err := grabber.GrabFromKubelet(ctx, nodeName)
framework.ExpectNoError(err)
@@ -497,12 +517,19 @@ func grabMetrics(ctx context.Context, grabber *e2emetrics.Grabber, nodeName stri
if len(samples) == 0 {
continue
}
// Find the *last* sample that has the label we are interested in.
for i := len(samples) - 1; i >= 0; i-- {
metricNameWithLabels := samples[i].Metric.String()
if metricNames.Has(metricNameWithLabels) {
// For each metric + label combination, remember the last sample
for i := range samples {
// E.g. "volume_manager_selinux_pod_context_mismatch_errors_total"
metricName := samples[i].Metric[testutil.MetricNameLabel]
if metricNames.Has(string(metricName)) {
// E.g. "volume_manager_selinux_pod_context_mismatch_errors_total{access_mode="RWOP",volume_plugin="kubernetes.io/csi/csi-mock-ns"}
metricNameWithLabels := samples[i].Metric.String()
// Filter out metrics of any other volume plugin
if strings.Contains(metricNameWithLabels, "volume_plugin=") && !strings.Contains(metricNameWithLabels, volumePluginLabel) {
continue
}
// Overwrite any previous value, so only the last one is stored.
metrics[metricNameWithLabels] = float64(samples[i].Value)
break
}
}
}
@@ -510,32 +537,21 @@ func grabMetrics(ctx context.Context, grabber *e2emetrics.Grabber, nodeName stri
return metrics, nil
}
func waitForMetricIncrease(ctx context.Context, grabber *e2emetrics.Grabber, nodeName string, allMetricNames, expectedIncreaseNames sets.String, initialValues map[string]float64, timeout time.Duration) error {
func waitForMetricIncrease(ctx context.Context, grabber *e2emetrics.Grabber, nodeName string, volumePluginLabel string, allMetricNames, expectedIncreaseNames sets.String, initialValues map[string]float64, timeout time.Duration) error {
var noIncreaseMetrics sets.String
var metrics map[string]float64
err := wait.Poll(time.Second, timeout, func() (bool, error) {
var err error
metrics, err = grabMetrics(ctx, grabber, nodeName, allMetricNames)
metrics, err = grabMetrics(ctx, grabber, nodeName, allMetricNames, volumePluginLabel)
if err != nil {
return false, err
}
noIncreaseMetrics = sets.NewString()
// Always evaluate all SELinux metrics to check that the other metrics are not unexpectedly increased.
for name := range allMetricNames {
expectIncrease := false
// allMetricNames can include "{volume_plugin="XXX"}", while expectedIncreaseNames does not.
// Compare them properly. Value of volume_plugin="XXX" was already checked in grabMetrics(),
// we can ignore it here.
for expectedIncreaseMetricName := range expectedIncreaseNames {
if strings.HasPrefix(name, expectedIncreaseMetricName) {
expectIncrease = true
break
}
}
if expectIncrease {
for name := range metrics {
if expectedIncreaseNames.Has(name) {
if metrics[name] <= initialValues[name] {
noIncreaseMetrics.Insert(name)
}
@@ -570,3 +586,22 @@ func dumpMetrics(metrics map[string]float64) {
framework.Logf("Metric %s: %v", key, metrics[key])
}
}
// Add labels to the metric name based on the current test case
func addLabels(metricNames sets.String, volumePluginLabel string, accessMode v1.PersistentVolumeAccessMode) sets.String {
ret := sets.NewString()
accessModeShortString := helper.GetAccessModesAsString([]v1.PersistentVolumeAccessMode{accessMode})
for metricName := range metricNames {
var metricWithLabels string
if metricsWithVolumePluginLabel.Has(metricName) {
metricWithLabels = fmt.Sprintf("%s{access_mode=\"%s\", %s}", metricName, accessModeShortString, volumePluginLabel)
} else {
metricWithLabels = fmt.Sprintf("%s{access_mode=\"%s\"}", metricName, accessModeShortString)
}
ret.Insert(metricWithLabels)
}
return ret
}