From 52558a031041d9708b608b4a3a09f4e9a8f7ee62 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 28 Feb 2024 10:53:01 +0100 Subject: [PATCH 1/4] Add e2e tests for SELinuxMount --- test/e2e/feature/feature.go | 10 ++- .../e2e/storage/csi_mock/csi_selinux_mount.go | 78 +++++++++++++++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go index b100ad67f8e..d71ad2e09cf 100644 --- a/test/e2e/feature/feature.go +++ b/test/e2e/feature/feature.go @@ -268,11 +268,15 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) SeccompDefault = framework.WithFeature(framework.ValidFeatures.Add("SeccompDefault")) - // TODO: document the feature (owning SIG, when to use this feature for a test) + // Owner: sig-storage + // This feature marks tests that need all schedulable Linux nodes in the cluster to have SELinux enabled. SELinux = framework.WithFeature(framework.ValidFeatures.Add("SELinux")) - // TODO: document the feature (owning SIG, when to use this feature for a test) - SELinuxMountReadWriteOncePod = framework.WithFeature(framework.ValidFeatures.Add("SELinuxMountReadWriteOncePod")) + // Owner: sig-storage + // This feature marks tests that need SELinuxMountReadWriteOncePod feature gate enabled and SELinuxMount **disabled**. + // This is a temporary feature to allow testing of metrics when SELinuxMount is disabled. + // TODO: remove when SELinuxMount feature gate is enabled by default. + SELinuxMountReadWriteOncePodOnly = framework.WithFeature(framework.ValidFeatures.Add("SELinuxMountReadWriteOncePodOnly")) // TODO: document the feature (owning SIG, when to use this feature for a test) ServiceCIDRs = framework.WithFeature(framework.ValidFeatures.Add("ServiceCIDRs")) diff --git a/test/e2e/storage/csi_mock/csi_selinux_mount.go b/test/e2e/storage/csi_mock/csi_selinux_mount.go index 5c97ec03646..5d357b773fe 100644 --- a/test/e2e/storage/csi_mock/csi_selinux_mount.go +++ b/test/e2e/storage/csi_mock/csi_selinux_mount.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" @@ -75,6 +76,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedFirstMountOptions []string expectedSecondMountOptions []string expectedUnstage bool + feature interface{} }{ // Start just a single pod and check its volume is mounted correctly { @@ -83,6 +85,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { firstPodSELinuxOpts: &seLinuxOpts1, volumeMode: v1.ReadWriteOncePod, expectedFirstMountOptions: []string{seLinuxMountOption1}, + feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), }, { name: "should add SELinux mount option to existing mount options", @@ -91,13 +94,23 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { mountOptions: []string{"noexec", "noatime"}, volumeMode: v1.ReadWriteOncePod, expectedFirstMountOptions: []string{"noexec", "noatime", seLinuxMountOption1}, + feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), }, { - name: "should not pass SELinux mount option for RWO volume", + name: "should not pass SELinux mount option for RWO volume with SELinuxMount disabled", csiDriverSELinuxEnabled: true, firstPodSELinuxOpts: &seLinuxOpts1, volumeMode: v1.ReadWriteOnce, expectedFirstMountOptions: nil, + feature: feature.SELinuxMountReadWriteOncePodOnly, + }, + { + name: "should pass SELinux mount option for RWO volume with SELinuxMount enabled", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + volumeMode: v1.ReadWriteOnce, + expectedFirstMountOptions: []string{seLinuxMountOption1}, + feature: framework.WithFeatureGate(features.SELinuxMount), }, { name: "should not pass SELinux mount option for Pod without SELinux context", @@ -105,6 +118,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { firstPodSELinuxOpts: nil, volumeMode: v1.ReadWriteOncePod, expectedFirstMountOptions: nil, + feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), }, { name: "should not pass SELinux mount option for CSI driver that does not support SELinux mount", @@ -112,10 +126,11 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { firstPodSELinuxOpts: &seLinuxOpts1, volumeMode: v1.ReadWriteOncePod, expectedFirstMountOptions: nil, + feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), }, // Start two pods in a sequence and check their volume is / is not unmounted in between { - name: "should not unstage volume when starting a second pod with the same SELinux context", + name: "should not unstage RWOP volume when starting a second pod with the same SELinux context", csiDriverSELinuxEnabled: true, firstPodSELinuxOpts: &seLinuxOpts1, startSecondPod: true, @@ -124,9 +139,10 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedFirstMountOptions: []string{seLinuxMountOption1}, expectedSecondMountOptions: []string{seLinuxMountOption1}, expectedUnstage: false, + feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), }, { - name: "should unstage volume when starting a second pod with different SELinux context", + name: "should unstage RWOP volume when starting a second pod with different SELinux context", csiDriverSELinuxEnabled: true, firstPodSELinuxOpts: &seLinuxOpts1, startSecondPod: true, @@ -135,11 +151,36 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedFirstMountOptions: []string{seLinuxMountOption1}, expectedSecondMountOptions: []string{seLinuxMountOption2}, expectedUnstage: true, + feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), + }, + { + name: "should not unstage RWO volume when starting a second pod with the same SELinux context", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + startSecondPod: true, + secondPodSELinuxOpts: &seLinuxOpts1, + volumeMode: v1.ReadWriteOnce, + expectedFirstMountOptions: []string{seLinuxMountOption1}, + expectedSecondMountOptions: []string{seLinuxMountOption1}, + expectedUnstage: false, + feature: framework.WithFeatureGate(features.SELinuxMount), + }, + { + name: "should unstage RWO volume when starting a second pod with different SELinux context", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + startSecondPod: true, + secondPodSELinuxOpts: &seLinuxOpts2, + volumeMode: v1.ReadWriteOnce, + expectedFirstMountOptions: []string{seLinuxMountOption1}, + expectedSecondMountOptions: []string{seLinuxMountOption2}, + expectedUnstage: true, + feature: framework.WithFeatureGate(features.SELinuxMount), }, } for _, t := range tests { t := t - ginkgo.It(t.name, func(ctx context.Context) { + ginkgo.It(t.name, t.feature, func(ctx context.Context) { if framework.NodeOSDistroIs("windows") { e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping") } @@ -252,7 +293,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { m := newMockDriverSetup(f) // [Serial]: the tests read global kube-controller-manager metrics, so no other test changes them in parallel. - f.Context("SELinuxMount metrics [LinuxOnly]", feature.SELinux, feature.SELinuxMountReadWriteOncePod, f.WithSerial(), func() { + f.Context("SELinuxMount metrics [LinuxOnly]", feature.SELinux, f.WithSerial(), func() { // Make sure all options are set so system specific defaults are not used. seLinuxOpts1 := v1.SELinuxOptions{ User: "system_u", @@ -276,6 +317,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { waitForSecondPodStart bool secondPodFailureEvent string expectIncreases sets.String + feature interface{} }{ { name: "warning is not bumped on two Pods with the same context on RWO volume", @@ -285,6 +327,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { volumeMode: v1.ReadWriteOnce, waitForSecondPodStart: true, expectIncreases: sets.NewString( /* no metric is increased, admitted_total was already increased when the first pod started */ ), + feature: feature.SELinuxMountReadWriteOncePodOnly, }, { name: "warning is bumped on two Pods with a different context on RWO volume", @@ -294,6 +337,28 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { volumeMode: v1.ReadWriteOnce, waitForSecondPodStart: true, expectIncreases: sets.NewString("volume_manager_selinux_volume_context_mismatch_warnings_total"), + feature: feature.SELinuxMountReadWriteOncePodOnly, + }, + { + name: "error is not bumped on two Pods with the same context on RWO volume and SELinuxMount enabled", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + secondPodSELinuxOpts: &seLinuxOpts1, + volumeMode: v1.ReadWriteOnce, + waitForSecondPodStart: true, + expectIncreases: sets.NewString( /* no metric is increased, admitted_total was already increased when the first pod started */ ), + feature: framework.WithFeatureGate(features.SELinuxMount), + }, + { + name: "error is bumped on two Pods with a different context on RWO volume and SELinuxMount enabled", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + secondPodSELinuxOpts: &seLinuxOpts2, + secondPodFailureEvent: "conflicting SELinux labels of volume", + volumeMode: v1.ReadWriteOnce, + waitForSecondPodStart: false, + expectIncreases: sets.NewString("volume_manager_selinux_volume_context_mismatch_errors_total"), + feature: framework.WithFeatureGate(features.SELinuxMount), }, { name: "error is bumped on two Pods with a different context on RWOP volume", @@ -304,11 +369,12 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { volumeMode: v1.ReadWriteOncePod, waitForSecondPodStart: false, expectIncreases: sets.NewString("volume_manager_selinux_volume_context_mismatch_errors_total"), + feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), }, } for _, t := range tests { t := t - ginkgo.It(t.name, func(ctx context.Context) { + ginkgo.It(t.name, t.feature, func(ctx context.Context) { // Some metrics use CSI driver name as a label, which is "csi-mock-" + the namespace name. volumePluginLabel := "{volume_plugin=\"kubernetes.io/csi/csi-mock-" + f.Namespace.Name + "\"}" From ba3562776f1564f89ca38cb6da6b5082e00d27f8 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 28 Feb 2024 10:53:09 +0100 Subject: [PATCH 2/4] Update SELinuxMount stage/unstage tests to allow RWO Previously, SELinuxMount started two pods and in laboratory conditions waited for the second Pod to get stuck (because of RWOP) and observed kubelet behavor after the test unstuck them (i.e. deleted the first Pod). When testing RWO volumes, the second Pod may not get stuck, it may actually run. So update the tests to allow the second Pod to run and start counting CSI calls for it earlier. --- .../e2e/storage/csi_mock/csi_selinux_mount.go | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/test/e2e/storage/csi_mock/csi_selinux_mount.go b/test/e2e/storage/csi_mock/csi_selinux_mount.go index 5d357b773fe..77ffe00dcca 100644 --- a/test/e2e/storage/csi_mock/csi_selinux_mount.go +++ b/test/e2e/storage/csi_mock/csi_selinux_mount.go @@ -180,7 +180,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { } for _, t := range tests { t := t - ginkgo.It(t.name, t.feature, func(ctx context.Context) { + f.It(t.name, t.feature, func(ctx context.Context) { if framework.NodeOSDistroIs("windows") { e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping") } @@ -218,6 +218,13 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { // Arrange 2nd part of the test ginkgo.By("Starting the second pod to check if a volume used by the initial pod is / is not unmounted based on SELinux context") + // count fresh CSI driver calls between the first and the second pod + nodeStageMountOpts = nil + nodePublishMountOpts = nil + unstageCalls.Store(0) + unpublishCalls.Store(0) + stageCalls.Store(0) + publishCalls.Store(0) // Skip scheduler, it would block scheduling the second pod with ReadWriteOncePod PV. pod, err = m.cs.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) @@ -231,31 +238,32 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { // DesiredStateOfWorld. // In this state, any volume UnPublish / UnStage must be done because of SELinux contexts and not // because of random races because volumes of the second pod are not in DesiredStateOfWorld yet. - ginkgo.By("Waiting for the second pod to fail to start because of ReadWriteOncePod.") - eventSelector := fields.Set{ - "involvedObject.kind": "Pod", - "involvedObject.name": pod2.Name, - "involvedObject.namespace": pod2.Namespace, - "reason": events.FailedMountVolume, - }.AsSelector().String() + ginkgo.By("Waiting for the second pod to start (or fail to start because of ReadWriteOncePod).") + reason := events.FailedMountVolume var msg string if t.expectedUnstage { // This message is emitted before kubelet checks for ReadWriteOncePod msg = "conflicting SELinux labels of volume" } else { - msg = "volume uses the ReadWriteOncePod access mode and is already in use by another pod" + // Kubelet should re-use staged volume. + if t.volumeMode == v1.ReadWriteOncePod { + // Wait for the second pod to get stuck because of RWOP. + msg = "volume uses the ReadWriteOncePod access mode and is already in use by another pod" + } else { + // There is nothing blocking the second pod from starting, wait for the second pod to fullly start. + reason = string(events.StartedContainer) + msg = "Started container" + } } + eventSelector := fields.Set{ + "involvedObject.kind": "Pod", + "involvedObject.name": pod2.Name, + "involvedObject.namespace": pod2.Namespace, + "reason": reason, + }.AsSelector().String() err = e2eevents.WaitTimeoutForEvent(ctx, m.cs, pod2.Namespace, eventSelector, msg, f.Timeouts.PodStart) framework.ExpectNoError(err, "waiting for event %q in the second test pod", msg) - // count fresh CSI driver calls between the first and the second pod - nodeStageMountOpts = nil - nodePublishMountOpts = nil - unstageCalls.Store(0) - unpublishCalls.Store(0) - stageCalls.Store(0) - publishCalls.Store(0) - // Act 2nd part of the test ginkgo.By("Deleting the initial pod") err = e2epod.DeletePodWithWait(ctx, m.cs, pod) @@ -374,7 +382,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { } for _, t := range tests { t := t - ginkgo.It(t.name, t.feature, func(ctx context.Context) { + f.It(t.name, t.feature, func(ctx context.Context) { // Some metrics use CSI driver name as a label, which is "csi-mock-" + the namespace name. volumePluginLabel := "{volume_plugin=\"kubernetes.io/csi/csi-mock-" + f.Namespace.Name + "\"}" From 2a22b6f6b8dff28bc2f5681066b381d9e38feb3f Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Wed, 28 Feb 2024 12:43:12 +0100 Subject: [PATCH 3/4] Add information about SELinux test tags --- test/e2e/storage/csi_mock/csi_selinux_mount.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/e2e/storage/csi_mock/csi_selinux_mount.go b/test/e2e/storage/csi_mock/csi_selinux_mount.go index 77ffe00dcca..e3d5af401a6 100644 --- a/test/e2e/storage/csi_mock/csi_selinux_mount.go +++ b/test/e2e/storage/csi_mock/csi_selinux_mount.go @@ -43,6 +43,19 @@ import ( admissionapi "k8s.io/pod-security-admission/api" ) +// Tests for SELinuxMount feature. +// KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1710-selinux-relabeling +// There are two feature gates: SELinuxMountReadWriteOncePod and SELinuxMount. +// These tags are used in the tests: +// +// [FeatureGate:SELinuxMountReadWriteOncePod] +// - The test requires SELinuxMountReadWriteOncePod enabled. SELinuxMount may be enabled or disabled, except for tests with Feature:SELinuxMountReadWriteOncePodOnly (see below). +// +// [FeatureGate:SELinuxMountReadWriteOncePod] [Feature:SELinuxMountReadWriteOncePodOnly] +// - The test requires SELinuxMountReadWriteOncePod enabled and SELinuxMount disabled. This checks metrics that are emitted only when SELinuxMount is disabled. +// +// [FeatureGate:SELinuxMount] +// - The test requires SELinuxMountReadWriteOncePod and SELinuxMount enabled. var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { f := framework.NewDefaultFramework("csi-mock-volumes-selinux") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged From 74417b517a029295aee4fd0a17792d813b74acf4 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 1 Mar 2024 14:37:54 +0100 Subject: [PATCH 4/4] Tag all feature gates required by a test Use all necessary feature gates in SELinuxMount tests. --- .../e2e/storage/csi_mock/csi_selinux_mount.go | 62 ++++++++++++------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/test/e2e/storage/csi_mock/csi_selinux_mount.go b/test/e2e/storage/csi_mock/csi_selinux_mount.go index e3d5af401a6..89f28514717 100644 --- a/test/e2e/storage/csi_mock/csi_selinux_mount.go +++ b/test/e2e/storage/csi_mock/csi_selinux_mount.go @@ -49,12 +49,12 @@ import ( // These tags are used in the tests: // // [FeatureGate:SELinuxMountReadWriteOncePod] -// - The test requires SELinuxMountReadWriteOncePod enabled. SELinuxMount may be enabled or disabled, except for tests with Feature:SELinuxMountReadWriteOncePodOnly (see below). +// - The test requires SELinuxMountReadWriteOncePod enabled. // // [FeatureGate:SELinuxMountReadWriteOncePod] [Feature:SELinuxMountReadWriteOncePodOnly] // - The test requires SELinuxMountReadWriteOncePod enabled and SELinuxMount disabled. This checks metrics that are emitted only when SELinuxMount is disabled. // -// [FeatureGate:SELinuxMount] +// [FeatureGate:SELinuxMountReadWriteOncePod] [FeatureGate:SELinuxMount] // - The test requires SELinuxMountReadWriteOncePod and SELinuxMount enabled. var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { f := framework.NewDefaultFramework("csi-mock-volumes-selinux") @@ -89,7 +89,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedFirstMountOptions []string expectedSecondMountOptions []string expectedUnstage bool - feature interface{} + testTags []interface{} }{ // Start just a single pod and check its volume is mounted correctly { @@ -98,7 +98,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { firstPodSELinuxOpts: &seLinuxOpts1, volumeMode: v1.ReadWriteOncePod, expectedFirstMountOptions: []string{seLinuxMountOption1}, - feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod)}, }, { name: "should add SELinux mount option to existing mount options", @@ -107,7 +107,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { mountOptions: []string{"noexec", "noatime"}, volumeMode: v1.ReadWriteOncePod, expectedFirstMountOptions: []string{"noexec", "noatime", seLinuxMountOption1}, - feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod)}, }, { name: "should not pass SELinux mount option for RWO volume with SELinuxMount disabled", @@ -115,7 +115,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { firstPodSELinuxOpts: &seLinuxOpts1, volumeMode: v1.ReadWriteOnce, expectedFirstMountOptions: nil, - feature: feature.SELinuxMountReadWriteOncePodOnly, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), feature.SELinuxMountReadWriteOncePodOnly}, }, { name: "should pass SELinux mount option for RWO volume with SELinuxMount enabled", @@ -123,7 +123,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { firstPodSELinuxOpts: &seLinuxOpts1, volumeMode: v1.ReadWriteOnce, expectedFirstMountOptions: []string{seLinuxMountOption1}, - feature: framework.WithFeatureGate(features.SELinuxMount), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, }, { name: "should not pass SELinux mount option for Pod without SELinux context", @@ -131,7 +131,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { firstPodSELinuxOpts: nil, volumeMode: v1.ReadWriteOncePod, expectedFirstMountOptions: nil, - feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod)}, }, { name: "should not pass SELinux mount option for CSI driver that does not support SELinux mount", @@ -139,7 +139,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { firstPodSELinuxOpts: &seLinuxOpts1, volumeMode: v1.ReadWriteOncePod, expectedFirstMountOptions: nil, - feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod)}, }, // Start two pods in a sequence and check their volume is / is not unmounted in between { @@ -152,7 +152,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedFirstMountOptions: []string{seLinuxMountOption1}, expectedSecondMountOptions: []string{seLinuxMountOption1}, expectedUnstage: false, - feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod)}, }, { name: "should unstage RWOP volume when starting a second pod with different SELinux context", @@ -164,7 +164,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedFirstMountOptions: []string{seLinuxMountOption1}, expectedSecondMountOptions: []string{seLinuxMountOption2}, expectedUnstage: true, - feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod)}, }, { name: "should not unstage RWO volume when starting a second pod with the same SELinux context", @@ -176,7 +176,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedFirstMountOptions: []string{seLinuxMountOption1}, expectedSecondMountOptions: []string{seLinuxMountOption1}, expectedUnstage: false, - feature: framework.WithFeatureGate(features.SELinuxMount), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, }, { name: "should unstage RWO volume when starting a second pod with different SELinux context", @@ -188,12 +188,12 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedFirstMountOptions: []string{seLinuxMountOption1}, expectedSecondMountOptions: []string{seLinuxMountOption2}, expectedUnstage: true, - feature: framework.WithFeatureGate(features.SELinuxMount), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, }, } for _, t := range tests { t := t - f.It(t.name, t.feature, func(ctx context.Context) { + testFunc := func(ctx context.Context) { if framework.NodeOSDistroIs("windows") { e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping") } @@ -303,7 +303,15 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { gomega.Expect(unpublishCalls.Load()).To(gomega.BeNumerically(">", 0), "NodeUnpublish calls after the first pod is deleted") gomega.Expect(publishCalls.Load()).To(gomega.BeNumerically(">", 0), "NodePublish calls for the second pod") gomega.Expect(nodePublishMountOpts).To(gomega.Equal(t.expectedSecondMountOptions), "NodePublish MountFlags for the second pod") - }) + } + // t.testTags is array and it's not possible to use It("name", func(){}, t.testTags...) + // Compose It() arguments separately. + args := []interface{}{ + t.name, + testFunc, + } + args = append(args, t.testTags...) + framework.It(args...) } }) }) @@ -338,7 +346,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { waitForSecondPodStart bool secondPodFailureEvent string expectIncreases sets.String - feature interface{} + testTags []interface{} }{ { name: "warning is not bumped on two Pods with the same context on RWO volume", @@ -348,7 +356,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { volumeMode: v1.ReadWriteOnce, waitForSecondPodStart: true, expectIncreases: sets.NewString( /* no metric is increased, admitted_total was already increased when the first pod started */ ), - feature: feature.SELinuxMountReadWriteOncePodOnly, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), feature.SELinuxMountReadWriteOncePodOnly}, }, { name: "warning is bumped on two Pods with a different context on RWO volume", @@ -358,7 +366,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { volumeMode: v1.ReadWriteOnce, waitForSecondPodStart: true, expectIncreases: sets.NewString("volume_manager_selinux_volume_context_mismatch_warnings_total"), - feature: feature.SELinuxMountReadWriteOncePodOnly, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), feature.SELinuxMountReadWriteOncePodOnly}, }, { name: "error is not bumped on two Pods with the same context on RWO volume and SELinuxMount enabled", @@ -368,7 +376,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { volumeMode: v1.ReadWriteOnce, waitForSecondPodStart: true, expectIncreases: sets.NewString( /* no metric is increased, admitted_total was already increased when the first pod started */ ), - feature: framework.WithFeatureGate(features.SELinuxMount), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, }, { name: "error is bumped on two Pods with a different context on RWO volume and SELinuxMount enabled", @@ -379,7 +387,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { volumeMode: v1.ReadWriteOnce, waitForSecondPodStart: false, expectIncreases: sets.NewString("volume_manager_selinux_volume_context_mismatch_errors_total"), - feature: framework.WithFeatureGate(features.SELinuxMount), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, }, { name: "error is bumped on two Pods with a different context on RWOP volume", @@ -390,12 +398,12 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { volumeMode: v1.ReadWriteOncePod, waitForSecondPodStart: false, expectIncreases: sets.NewString("volume_manager_selinux_volume_context_mismatch_errors_total"), - feature: framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod)}, }, } for _, t := range tests { t := t - f.It(t.name, t.feature, func(ctx context.Context) { + testFunc := func(ctx context.Context) { // Some metrics use CSI driver name as a label, which is "csi-mock-" + the namespace name. volumePluginLabel := "{volume_plugin=\"kubernetes.io/csi/csi-mock-" + f.Namespace.Name + "\"}" @@ -467,7 +475,15 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { ginkgo.By("Waiting for expected metric changes") err = waitForMetricIncrease(ctx, grabber, pod.Spec.NodeName, allMetrics, t.expectIncreases, metrics, framework.PodStartShortTimeout) framework.ExpectNoError(err, "waiting for metrics %s to increase", t.expectIncreases) - }) + } + // t.testTags is array and it's not possible to use It("name", func(){xxx}, t.testTags...) + // Compose It() arguments separately. + args := []interface{}{ + t.name, + testFunc, + } + args = append(args, t.testTags...) + framework.It(args...) } }) })