From c478e4bd38217577a871c4103b36d3a0cff6ba8b Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Thu, 10 Oct 2024 15:41:40 +0200 Subject: [PATCH] Add e2e tests --- test/e2e/storage/csimock/base.go | 9 +- test/e2e/storage/csimock/csi_selinux_mount.go | 197 +++++++++++++++++- 2 files changed, 193 insertions(+), 13 deletions(-) diff --git a/test/e2e/storage/csimock/base.go b/test/e2e/storage/csimock/base.go index dcad86bd06a..1f5924d53f8 100644 --- a/test/e2e/storage/csimock/base.go +++ b/test/e2e/storage/csimock/base.go @@ -465,7 +465,7 @@ func (m *mockDriverSetup) createPodWithFSGroup(ctx context.Context, fsGroup *int return class, claim, pod } -func (m *mockDriverSetup) createPodWithSELinux(ctx context.Context, accessModes []v1.PersistentVolumeAccessMode, mountOptions []string, seLinuxOpts *v1.SELinuxOptions) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { +func (m *mockDriverSetup) createPodWithSELinux(ctx context.Context, accessModes []v1.PersistentVolumeAccessMode, mountOptions []string, seLinuxOpts *v1.SELinuxOptions, policy *v1.PodSELinuxChangePolicy) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { ginkgo.By("Creating pod with SELinux context") f := m.f nodeSelection := m.config.ClientNodeSelection @@ -482,7 +482,7 @@ func (m *mockDriverSetup) createPodWithSELinux(ctx context.Context, accessModes ReclaimPolicy: m.tp.reclaimPolicy, } class, claim := createClaim(ctx, f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name, accessModes) - pod, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, seLinuxOpts) + pod, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, seLinuxOpts, policy) framework.ExpectNoError(err, "Failed to create pause pod with SELinux context %s: %v", seLinuxOpts, err) if class != nil { @@ -804,14 +804,15 @@ func startBusyBoxPodWithVolumeSource(cs clientset.Interface, volumeSource v1.Vol return cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) } -func startPausePodWithSELinuxOptions(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection, ns string, seLinuxOpts *v1.SELinuxOptions) (*v1.Pod, error) { +func startPausePodWithSELinuxOptions(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection, ns string, seLinuxOpts *v1.SELinuxOptions, policy *v1.PodSELinuxChangePolicy) (*v1.Pod, error) { pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "pvc-volume-tester-", }, Spec: v1.PodSpec{ SecurityContext: &v1.PodSecurityContext{ - SELinuxOptions: seLinuxOpts, + SELinuxOptions: seLinuxOpts, + SELinuxChangePolicy: policy, }, Containers: []v1.Container{ { diff --git a/test/e2e/storage/csimock/csi_selinux_mount.go b/test/e2e/storage/csimock/csi_selinux_mount.go index c1554014f2d..1191c7b363c 100644 --- a/test/e2e/storage/csimock/csi_selinux_mount.go +++ b/test/e2e/storage/csimock/csi_selinux_mount.go @@ -47,7 +47,7 @@ import ( // Tests for SELinuxMount feature. // KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1710-selinux-relabeling -// There are two feature gates: SELinuxMountReadWriteOncePod and SELinuxMount. +// There are three feature gates: SELinuxMountReadWriteOncePod, SELinuxChangePolicy and SELinuxMount. // These tags are used in the tests: // // [FeatureGate:SELinuxMountReadWriteOncePod] @@ -56,12 +56,19 @@ import ( // [FeatureGate:SELinuxMountReadWriteOncePod] [Feature:SELinuxMountReadWriteOncePodOnly] // - The test requires SELinuxMountReadWriteOncePod enabled and SELinuxMount disabled. This checks metrics that are emitted only when SELinuxMount is disabled. // -// [FeatureGate:SELinuxMountReadWriteOncePod] [FeatureGate:SELinuxMount] -// - The test requires SELinuxMountReadWriteOncePod and SELinuxMount enabled. +// [FeatureGate:SELinuxMountReadWriteOncePod] [Feature:SELinuxMountReadWriteOncePodOnly] [Feature:SELinuxChangePolicy] +// - The test requires SELinuxMountReadWriteOncePod and SELinuxChangePolicy enabled and SELinuxMount disabled. This checks metrics that are emitted only when SELinuxMount is disabled. +// +// [FeatureGate:SELinuxMountReadWriteOncePod] [Feature:SELinuxChangePolicy] [FeatureGate:SELinuxMount] +// - The test requires SELinuxMountReadWriteOncePod, Feature:SELinuxChangePolicy and SELinuxMount enabled. +// +// All other feature gate combinations should be invalid. var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { f := framework.NewDefaultFramework("csi-mock-volumes-selinux") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged m := newMockDriverSetup(f) + recursive := v1.SELinuxChangePolicyRecursive + mount := v1.SELinuxChangePolicyMountOption f.Context("SELinuxMount [LinuxOnly]", feature.SELinux, func() { // Make sure all options are set so system specific defaults are not used. @@ -84,8 +91,10 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { name string csiDriverSELinuxEnabled bool firstPodSELinuxOpts *v1.SELinuxOptions + firstPodChangePolicy *v1.PodSELinuxChangePolicy startSecondPod bool secondPodSELinuxOpts *v1.SELinuxOptions + secondPodChangePolicy *v1.PodSELinuxChangePolicy mountOptions []string volumeMode v1.PersistentVolumeAccessMode expectedFirstMountOptions []string @@ -120,12 +129,38 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), feature.SELinuxMountReadWriteOncePodOnly}, }, { - name: "should pass SELinux mount option for RWO volume with SELinuxMount enabled", + name: "should not pass SELinux mount option for RWO volume with only SELinuxChangePolicy enabled", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + volumeMode: v1.ReadWriteOnce, + expectedFirstMountOptions: nil, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), feature.SELinuxMountReadWriteOncePodOnly, framework.WithFeatureGate(features.SELinuxChangePolicy)}, + }, + { + name: "should pass SELinux mount option for RWO volume with SELinuxMount enabled and nil policy", csiDriverSELinuxEnabled: true, firstPodSELinuxOpts: &seLinuxOpts1, volumeMode: v1.ReadWriteOnce, expectedFirstMountOptions: []string{seLinuxMountOption1}, - testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxChangePolicy), framework.WithFeatureGate(features.SELinuxMount)}, + }, + { + name: "should pass SELinux mount option for RWO volume with SELinuxMount enabled and MountOption policy", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &mount, + volumeMode: v1.ReadWriteOnce, + expectedFirstMountOptions: []string{seLinuxMountOption1}, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxChangePolicy), framework.WithFeatureGate(features.SELinuxMount)}, + }, + { + name: "should not pass SELinux mount option for RWO volume with SELinuxMount disabled and Recursive policy", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &recursive, + volumeMode: v1.ReadWriteOnce, + expectedFirstMountOptions: nil, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxChangePolicy), framework.WithFeatureGate(features.SELinuxMount)}, }, { name: "should not pass SELinux mount option for Pod without SELinux context", @@ -192,6 +227,34 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { expectedUnstage: true, testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, }, + { + name: "should unstage RWO volume when starting a second pod with different policy (MountOption -> Recursive)", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &mount, + startSecondPod: true, + secondPodSELinuxOpts: &seLinuxOpts2, + secondPodChangePolicy: &recursive, + volumeMode: v1.ReadWriteOnce, + expectedFirstMountOptions: []string{seLinuxMountOption1}, + expectedSecondMountOptions: nil, + expectedUnstage: true, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, + }, + { + name: "should unstage RWO volume when starting a second pod with different policy (Recursive -> MountOption)", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &recursive, + startSecondPod: true, + secondPodSELinuxOpts: &seLinuxOpts2, + secondPodChangePolicy: &mount, + volumeMode: v1.ReadWriteOnce, + expectedFirstMountOptions: nil, + expectedSecondMountOptions: []string{seLinuxMountOption2}, + expectedUnstage: true, + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, + }, } for _, t := range tests { t := t @@ -212,7 +275,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { // Act ginkgo.By("Starting the initial pod") accessModes := []v1.PersistentVolumeAccessMode{t.volumeMode} - _, claim, pod := m.createPodWithSELinux(ctx, accessModes, t.mountOptions, t.firstPodSELinuxOpts) + _, claim, pod := m.createPodWithSELinux(ctx, accessModes, t.mountOptions, t.firstPodSELinuxOpts, t.firstPodChangePolicy) err := e2epod.WaitForPodNameRunningInNamespace(ctx, m.cs, pod.Name, pod.Namespace) framework.ExpectNoError(err, "starting the initial pod") @@ -245,7 +308,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { pod, err = m.cs.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "getting the initial pod") nodeSelection := e2epod.NodeSelection{Name: pod.Spec.NodeName} - pod2, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, t.secondPodSELinuxOpts) + pod2, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, t.secondPodSELinuxOpts, t.secondPodChangePolicy) framework.ExpectNoError(err, "creating second pod with SELinux context %s", t.secondPodSELinuxOpts) m.pods = append(m.pods, pod2) @@ -356,12 +419,16 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { Type: "container_t", Level: "s0:c98,c99", } + recursive := v1.SELinuxChangePolicyRecursive + mount := v1.SELinuxChangePolicyMountOption tests := []struct { name string csiDriverSELinuxEnabled bool firstPodSELinuxOpts *v1.SELinuxOptions + firstPodChangePolicy *v1.PodSELinuxChangePolicy secondPodSELinuxOpts *v1.SELinuxOptions + secondPodChangePolicy *v1.PodSELinuxChangePolicy volumeMode v1.PersistentVolumeAccessMode waitForSecondPodStart bool secondPodFailureEvent string @@ -388,6 +455,30 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_warnings_total"), testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), feature.SELinuxMountReadWriteOncePodOnly}, }, + { + name: "warning is bumped on two Pods with different policies on RWO volume", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: nil, + secondPodSELinuxOpts: &seLinuxOpts1, + secondPodChangePolicy: &recursive, + volumeMode: v1.ReadWriteOnce, + waitForSecondPodStart: true, + expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_warnings_total"), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), feature.SELinuxMountReadWriteOncePodOnly}, + }, + { + name: "warning is not bumped on two Pods with Recursive policy and a different context on RWO volume", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &recursive, + secondPodSELinuxOpts: &seLinuxOpts2, + secondPodChangePolicy: &recursive, + volumeMode: v1.ReadWriteOnce, + waitForSecondPodStart: true, + expectIncreases: sets.New[string]( /* no metric is increased, admitted_total was already increased when the first pod started */ ), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxChangePolicy), feature.SELinuxMountReadWriteOncePodOnly}, + }, { name: "error is not bumped on two Pods with the same context on RWO volume and SELinuxMount enabled", csiDriverSELinuxEnabled: true, @@ -409,6 +500,45 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_errors_total"), testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, }, + { + name: "error is bumped on two Pods with a different policy on RWO volume and SELinuxMount enabled (nil + Recursive)", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: nil, + secondPodSELinuxOpts: &seLinuxOpts1, + secondPodChangePolicy: &recursive, + secondPodFailureEvent: "conflicting SELinux labels of volume", + volumeMode: v1.ReadWriteOnce, + waitForSecondPodStart: false, + expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_errors_total"), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, + }, + { + name: "error is bumped on two Pods with a different policy on RWO volume and SELinuxMount enabled (Recursive + nil)", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &recursive, + secondPodSELinuxOpts: &seLinuxOpts1, + secondPodChangePolicy: nil, + secondPodFailureEvent: "conflicting SELinux labels of volume", + volumeMode: v1.ReadWriteOnce, + waitForSecondPodStart: false, + expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_errors_total"), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, + }, + { + name: "error is bumped on two Pods with a different policy on RWO volume and SELinuxMount enabled (Recursive + MountOption)", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &recursive, + secondPodSELinuxOpts: &seLinuxOpts1, + secondPodChangePolicy: &mount, + secondPodFailureEvent: "conflicting SELinux labels of volume", + volumeMode: v1.ReadWriteOnce, + waitForSecondPodStart: false, + expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_errors_total"), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, + }, { name: "error is bumped on two Pods with a different context on RWX volume and SELinuxMount enabled", csiDriverSELinuxEnabled: true, @@ -420,6 +550,42 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_errors_total"), testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, }, + { + name: "error is not bumped on two Pods with Recursive policy and a different context on RWX volume", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &recursive, + secondPodSELinuxOpts: &seLinuxOpts2, + secondPodChangePolicy: &recursive, + volumeMode: v1.ReadWriteMany, + waitForSecondPodStart: true, + expectIncreases: sets.New[string]( /* no metric is increased, admitted_total was already increased when the first pod started */ ), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxChangePolicy), framework.WithFeatureGate(features.SELinuxMount)}, + }, + { + name: "error is not bumped on two Pods with a different policy RWX volume (nil + MountOption", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &mount, + secondPodSELinuxOpts: &seLinuxOpts1, + secondPodChangePolicy: nil, + volumeMode: v1.ReadWriteMany, + waitForSecondPodStart: true, + expectIncreases: sets.New[string]( /* no metric is increased, admitted_total was already increased when the first pod started */ ), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxChangePolicy), framework.WithFeatureGate(features.SELinuxMount)}, + }, + { + name: "error is not bumped on two Pods with a different policy RWX volume (MountOption + MountOption", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &mount, + secondPodSELinuxOpts: &seLinuxOpts1, + secondPodChangePolicy: &mount, + volumeMode: v1.ReadWriteMany, + waitForSecondPodStart: true, + expectIncreases: sets.New[string]( /* no metric is increased, admitted_total was already increased when the first pod started */ ), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxChangePolicy), framework.WithFeatureGate(features.SELinuxMount)}, + }, { name: "error is bumped on two Pods with a different context on RWOP volume", csiDriverSELinuxEnabled: true, @@ -431,6 +597,19 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_errors_total"), testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod)}, }, + { + name: "error is bumped on two Pods with MountOption policy and a different context on RWOP volume", + csiDriverSELinuxEnabled: true, + firstPodSELinuxOpts: &seLinuxOpts1, + firstPodChangePolicy: &mount, + secondPodSELinuxOpts: &seLinuxOpts2, + secondPodChangePolicy: &mount, + secondPodFailureEvent: "conflicting SELinux labels of volume", + volumeMode: v1.ReadWriteOncePod, + waitForSecondPodStart: false, + expectIncreases: sets.New[string]("volume_manager_selinux_volume_context_mismatch_errors_total"), + testTags: []interface{}{framework.WithFeatureGate(features.SELinuxMountReadWriteOncePod), framework.WithFeatureGate(features.SELinuxMount)}, + }, } for _, t := range tests { t := t @@ -456,7 +635,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { ginkgo.By("Starting the first pod") accessModes := []v1.PersistentVolumeAccessMode{t.volumeMode} - _, claim, pod := m.createPodWithSELinux(ctx, accessModes, []string{}, t.firstPodSELinuxOpts) + _, claim, pod := m.createPodWithSELinux(ctx, accessModes, []string{}, t.firstPodSELinuxOpts, t.firstPodChangePolicy) err = e2epod.WaitForPodNameRunningInNamespace(ctx, m.cs, pod.Name, pod.Namespace) framework.ExpectNoError(err, "starting the initial pod") @@ -471,7 +650,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics", func() { ginkgo.By("Starting the second pod") // Skip scheduler, it would block scheduling the second pod with ReadWriteOncePod PV. nodeSelection := e2epod.NodeSelection{Name: pod.Spec.NodeName} - pod2, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, t.secondPodSELinuxOpts) + pod2, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, t.secondPodSELinuxOpts, t.secondPodChangePolicy) framework.ExpectNoError(err, "creating second pod with SELinux context %s", t.secondPodSELinuxOpts) m.pods = append(m.pods, pod2)