From d6ee9ca860b7ee1b1cb3dfe8d108a1c2cbbc681c Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Mon, 12 Feb 2024 17:09:13 -0500 Subject: [PATCH 1/3] test/e2e_node: add proc mount tests including one Alpha only test, as the feature is in alpha Signed-off-by: Peter Hunt Co-authored-by: Sohan Kunkerkar --- test/e2e/nodefeature/nodefeature.go | 5 ++ test/e2e_node/proc_mount_test.go | 115 ++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 test/e2e_node/proc_mount_test.go diff --git a/test/e2e/nodefeature/nodefeature.go b/test/e2e/nodefeature/nodefeature.go index 964e7aefa85..ffa53ce8d42 100644 --- a/test/e2e/nodefeature/nodefeature.go +++ b/test/e2e/nodefeature/nodefeature.go @@ -88,6 +88,9 @@ var ( // RecursiveReadOnlyMounts (SIG-node, used for testing recursive read-only mounts ) RecursiveReadOnlyMounts = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("RecursiveReadOnlyMounts")) + // TODO: document the feature (owning SIG, when to use this feature for a test) + ProcMountType = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("ProcMountType")) + // TODO: document the feature (owning SIG, when to use this feature for a test) ResourceMetrics = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("ResourceMetrics")) @@ -104,6 +107,8 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) SystemNodeCriticalPod = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("SystemNodeCriticalPod")) + // TODO: document the feature (owning SIG, when to use this feature for a test) + UserNamespacesSupport = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("UserNamespacesSupport")) // Please keep the list in alphabetical order. ) diff --git a/test/e2e_node/proc_mount_test.go b/test/e2e_node/proc_mount_test.go new file mode 100644 index 00000000000..50df27ceed3 --- /dev/null +++ b/test/e2e_node/proc_mount_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "context" + "strings" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + gomegatypes "github.com/onsi/gomega/types" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/feature" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e/nodefeature" + testutils "k8s.io/kubernetes/test/utils" + imageutils "k8s.io/kubernetes/test/utils/image" + admissionapi "k8s.io/pod-security-admission/api" +) + +var falseVar = false + +var _ = SIGDescribe("DefaultProcMount [LinuxOnly]", framework.WithNodeConformance(), func() { + f := framework.NewDefaultFramework("proc-mount-default-test") + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + + ginkgo.It("will mask proc mounts by default", func(ctx context.Context) { + testProcMount(ctx, f, v1.DefaultProcMount, gomega.BeNumerically(">=", 10), gomega.BeNumerically(">=", 7)) + }) +}) + +var _ = SIGDescribe("ProcMount [LinuxOnly]", nodefeature.ProcMountType, nodefeature.UserNamespacesSupport, feature.UserNamespacesSupport, func() { + f := framework.NewDefaultFramework("proc-mount-baseline-test") + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + + f.It("will fail to unmask proc mounts if not privileged", func(ctx context.Context) { + pmt := v1.UnmaskedProcMount + podClient := e2epod.NewPodClient(f) + _, err := podClient.PodInterface.Create(ctx, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "proc-mount-pod"}, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-1", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"/bin/sleep"}, + Args: []string{"10000"}, + SecurityContext: &v1.SecurityContext{ + ProcMount: &pmt, + }, + }, + }, + HostUsers: &falseVar, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) +}) + +var _ = SIGDescribe("ProcMount [LinuxOnly]", nodefeature.ProcMountType, nodefeature.UserNamespacesSupport, feature.UserNamespacesSupport, func() { + f := framework.NewDefaultFramework("proc-mount-privileged-test") + + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + f.It("will unmask proc mounts if requested", func(ctx context.Context) { + testProcMount(ctx, f, v1.UnmaskedProcMount, gomega.Equal(1), gomega.BeZero()) + }) +}) + +func testProcMount(ctx context.Context, f *framework.Framework, pmt v1.ProcMountType, expectedLines gomegatypes.GomegaMatcher, expectedReadOnly gomegatypes.GomegaMatcher) { + ginkgo.By("creating a target pod") + podClient := e2epod.NewPodClient(f) + pod := podClient.CreateSync(ctx, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "proc-mount-pod"}, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container-1", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"/bin/sleep"}, + Args: []string{"10000"}, + SecurityContext: &v1.SecurityContext{ + ProcMount: &pmt, + }, + }, + }, + HostUsers: &falseVar, + }, + }) + + _, err := testutils.PodRunningReady(pod) + framework.ExpectNoError(err) + + output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", "mount | grep /proc") + ginkgo.By(output) + lines := strings.Split(output, "\n") + gomega.Expect(len(lines)).To(expectedLines) + gomega.Expect(strings.Count(output, "(ro")).To(expectedReadOnly) +} From 4f57a1448d09330223a3a640b6f5d4c6df922753 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Wed, 14 Feb 2024 11:49:15 -0500 Subject: [PATCH 2/3] test/e2e: add test to ensure proc mount is stripped from windows pods Signed-off-by: Peter Hunt --- test/e2e/windows/security_context.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/test/e2e/windows/security_context.go b/test/e2e/windows/security_context.go index ebb62dbbaa0..a793859fa72 100644 --- a/test/e2e/windows/security_context.go +++ b/test/e2e/windows/security_context.go @@ -136,7 +136,7 @@ var _ = sigDescribe(feature.Windows, "SecurityContext", skipUnlessWindows(func() e2eoutput.TestContainerOutput(ctx, f, "check pod SecurityContext username", pod, 1, []string{"ContainerAdministrator"}) }) - ginkgo.It("should ignore Linux Specific SecurityContext if set", func(ctx context.Context) { + ginkgo.It("should ignore SELinux Specific SecurityContext if set", func(ctx context.Context) { ginkgo.By("Creating a pod with SELinux options") // It is sufficient to show that the pod comes up here. Since we're stripping the SELinux and other linux // security contexts in apiserver and not updating the pod object in the apiserver, we cannot validate the @@ -160,6 +160,30 @@ var _ = sigDescribe(feature.Windows, "SecurityContext", skipUnlessWindows(func() f.Namespace.Name), "failed to wait for pod %s to be running", windowsPodWithSELinux.Name) }) + ginkgo.It("should ignore ProcMount Specific SecurityContext if set", func(ctx context.Context) { + ginkgo.By("Creating a pod with ProcMount options") + // It is sufficient to show that the pod comes up here. Since we're stripping the SELinux and other linux + // security contexts in apiserver and not updating the pod object in the apiserver, we cannot validate the + // pod object to not have those security contexts. However the pod coming to running state is a sufficient + // enough condition for us to validate since prior to https://github.com/kubernetes/kubernetes/pull/93475 + // the pod would have failed to come up. + windowsPodWithSELinux := createTestPod(f, imageutils.GetE2EImage(imageutils.Agnhost), windowsOS) + windowsPodWithSELinux.Spec.Containers[0].Args = []string{"test-webserver-with-selinux"} + windowsPodWithSELinux.Spec.SecurityContext = &v1.PodSecurityContext{} + pmt := v1.UnmaskedProcMount + containerUserName := "ContainerAdministrator" + windowsPodWithSELinux.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ + ProcMount: &pmt, + WindowsOptions: &v1.WindowsSecurityContextOptions{RunAsUserName: &containerUserName}} + windowsPodWithSELinux.Spec.Tolerations = []v1.Toleration{{Key: "os", Value: "Windows"}} + windowsPodWithSELinux, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, + windowsPodWithSELinux, metav1.CreateOptions{}) + framework.ExpectNoError(err) + framework.Logf("Created pod %v", windowsPodWithSELinux) + framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, windowsPodWithSELinux.Name, + f.Namespace.Name), "failed to wait for pod %s to be running", windowsPodWithSELinux.Name) + }) + ginkgo.It("should not be able to create pods with containers running as ContainerAdministrator when runAsNonRoot is true", func(ctx context.Context) { ginkgo.By("Creating a pod") From a20a8225cfcbf042b671d741e881125579683a4a Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Mon, 4 Mar 2024 15:41:41 -0500 Subject: [PATCH 3/3] e2e_node: skip proc mount tests on nodes without userns support in the runtime Signed-off-by: Peter Hunt Co-authored-by: Sohan Kunkerkar --- test/e2e_node/proc_mount_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/e2e_node/proc_mount_test.go b/test/e2e_node/proc_mount_test.go index 50df27ceed3..dede318adbb 100644 --- a/test/e2e_node/proc_mount_test.go +++ b/test/e2e_node/proc_mount_test.go @@ -28,6 +28,7 @@ import ( "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/nodefeature" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -50,6 +51,9 @@ var _ = SIGDescribe("ProcMount [LinuxOnly]", nodefeature.ProcMountType, nodefeat f.NamespacePodSecurityLevel = admissionapi.LevelBaseline f.It("will fail to unmask proc mounts if not privileged", func(ctx context.Context) { + if !supportsUserNS(ctx, f) { + e2eskipper.Skipf("runtime does not support user namespaces") + } pmt := v1.UnmaskedProcMount podClient := e2epod.NewPodClient(f) _, err := podClient.PodInterface.Create(ctx, &v1.Pod{ @@ -79,6 +83,9 @@ var _ = SIGDescribe("ProcMount [LinuxOnly]", nodefeature.ProcMountType, nodefeat f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged f.It("will unmask proc mounts if requested", func(ctx context.Context) { + if !supportsUserNS(ctx, f) { + e2eskipper.Skipf("runtime does not support user namespaces") + } testProcMount(ctx, f, v1.UnmaskedProcMount, gomega.Equal(1), gomega.BeZero()) }) }) @@ -113,3 +120,17 @@ func testProcMount(ctx context.Context, f *framework.Framework, pmt v1.ProcMount gomega.Expect(len(lines)).To(expectedLines) gomega.Expect(strings.Count(output, "(ro")).To(expectedReadOnly) } + +func supportsUserNS(ctx context.Context, f *framework.Framework) bool { + nodeList, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + framework.ExpectNoError(err) + // Assuming that there is only one node, because this is a node e2e test. + gomega.Expect(nodeList.Items).To(gomega.HaveLen(1)) + node := nodeList.Items[0] + for _, rc := range node.Status.RuntimeHandlers { + if rc.Name == "" && rc.Features != nil && *rc.Features.UserNamespaces { + return true + } + } + return false +}