diff --git a/test/e2e/common/node/pods.go b/test/e2e/common/node/pods.go index a35657bfa0a..f9e2ee6dc21 100644 --- a/test/e2e/common/node/pods.go +++ b/test/e2e/common/node/pods.go @@ -184,7 +184,7 @@ func expectNoErrorWithRetries(fn func() error, maxRetries int, explain ...interf var _ = SIGDescribe("Pods", func() { f := framework.NewDefaultFramework("pods") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelRestricted var podClient *framework.PodClient var dc dynamic.Interface @@ -200,7 +200,7 @@ var _ = SIGDescribe("Pods", func() { */ framework.ConformanceIt("should get a host IP [NodeConformance]", func() { name := "pod-hostip-" + string(uuid.NewUUID()) - testHostIP(podClient, &v1.Pod{ + testHostIP(podClient, e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, @@ -212,7 +212,7 @@ var _ = SIGDescribe("Pods", func() { }, }, }, - }) + })) }) /* @@ -224,7 +224,7 @@ var _ = SIGDescribe("Pods", func() { ginkgo.By("creating the pod") name := "pod-submit-remove-" + string(uuid.NewUUID()) value := strconv.Itoa(time.Now().Nanosecond()) - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: map[string]string{ @@ -235,12 +235,12 @@ var _ = SIGDescribe("Pods", func() { Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "nginx", - Image: imageutils.GetE2EImage(imageutils.Nginx), + Name: "pause", + Image: imageutils.GetPauseImageName(), }, }, }, - } + }) ginkgo.By("setting up watch") selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) @@ -342,7 +342,7 @@ var _ = SIGDescribe("Pods", func() { ginkgo.By("creating the pod") name := "pod-update-" + string(uuid.NewUUID()) value := strconv.Itoa(time.Now().Nanosecond()) - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: map[string]string{ @@ -353,12 +353,12 @@ var _ = SIGDescribe("Pods", func() { Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "nginx", - Image: imageutils.GetE2EImage(imageutils.Nginx), + Name: "pause", + Image: imageutils.GetPauseImageName(), }, }, }, - } + }) ginkgo.By("submitting the pod to kubernetes") pod = podClient.CreateSync(pod) @@ -396,7 +396,7 @@ var _ = SIGDescribe("Pods", func() { ginkgo.By("creating the pod") name := "pod-update-activedeadlineseconds-" + string(uuid.NewUUID()) value := strconv.Itoa(time.Now().Nanosecond()) - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: map[string]string{ @@ -407,18 +407,18 @@ var _ = SIGDescribe("Pods", func() { Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "nginx", - Image: imageutils.GetE2EImage(imageutils.Nginx), + Name: "pause", + Image: imageutils.GetPauseImageName(), }, }, }, - } + }) ginkgo.By("submitting the pod to kubernetes") podClient.CreateSync(pod) ginkgo.By("verifying the pod is in kubernetes") - selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) + selector := labels.SelectorFromSet(labels.Set{"time": value}) options := metav1.ListOptions{LabelSelector: selector.String()} pods, err := podClient.List(context.TODO(), options) framework.ExpectNoError(err, "failed to query for pods") @@ -442,7 +442,7 @@ var _ = SIGDescribe("Pods", func() { // Make a pod that will be a service. // This pod serves its hostname via HTTP. serverName := "server-envvars-" + string(uuid.NewUUID()) - serverPod := &v1.Pod{ + serverPod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: serverName, Labels: map[string]string{"name": serverName}, @@ -456,7 +456,7 @@ var _ = SIGDescribe("Pods", func() { }, }, }, - } + }) podClient.CreateSync(serverPod) // This service exposes port 8080 of the test pod as a service on port 8765 @@ -490,7 +490,7 @@ var _ = SIGDescribe("Pods", func() { // Make a client pod that verifies that it has the service environment variables. podName := "client-envvars-" + string(uuid.NewUUID()) const containerName = "env3cont" - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Labels: map[string]string{"name": podName}, @@ -505,7 +505,7 @@ var _ = SIGDescribe("Pods", func() { }, RestartPolicy: v1.RestartPolicyNever, }, - } + }) // It's possible for the Pod to be created before the Kubelet is updated with the new // service. In that case, we just retry. @@ -536,7 +536,7 @@ var _ = SIGDescribe("Pods", func() { ginkgo.By("creating the pod") name := "pod-exec-websocket-" + string(uuid.NewUUID()) - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, @@ -549,7 +549,7 @@ var _ = SIGDescribe("Pods", func() { }, }, }, - } + }) ginkgo.By("submitting the pod to kubernetes") pod = podClient.CreateSync(pod) @@ -618,7 +618,7 @@ var _ = SIGDescribe("Pods", func() { ginkgo.By("creating the pod") name := "pod-logs-websocket-" + string(uuid.NewUUID()) - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, @@ -631,7 +631,7 @@ var _ = SIGDescribe("Pods", func() { }, }, }, - } + }) ginkgo.By("submitting the pod to kubernetes") podClient.CreateSync(pod) @@ -673,7 +673,7 @@ var _ = SIGDescribe("Pods", func() { ginkgo.It("should have their auto-restart back-off timer reset on image update [Slow][NodeConformance]", func() { podName := "pod-back-off-image" containerName := "back-off" - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Labels: map[string]string{"test": "back-off-image"}, @@ -687,7 +687,7 @@ var _ = SIGDescribe("Pods", func() { }, }, }, - } + }) delay1, delay2 := startPodAndGetBackOffs(podClient, pod, buildBackOffDuration) @@ -714,7 +714,7 @@ var _ = SIGDescribe("Pods", func() { ginkgo.It("should cap back-off at MaxContainerBackOff [Slow][NodeConformance]", func() { podName := "back-off-cap" containerName := "back-off-cap" - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Labels: map[string]string{"test": "liveness"}, @@ -728,7 +728,7 @@ var _ = SIGDescribe("Pods", func() { }, }, }, - } + }) podClient.CreateSync(pod) time.Sleep(2 * kubelet.MaxContainerBackOff) // it takes slightly more than 2*x to get to a back-off of x @@ -770,7 +770,7 @@ var _ = SIGDescribe("Pods", func() { readinessGate1 := "k8s.io/test-condition1" readinessGate2 := "k8s.io/test-condition2" patchStatusFmt := `{"status":{"conditions":[{"type":%q, "status":%q}]}}` - pod := &v1.Pod{ + pod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Labels: map[string]string{"test": "pod-readiness-gate"}, @@ -788,7 +788,7 @@ var _ = SIGDescribe("Pods", func() { {ConditionType: v1.PodConditionType(readinessGate2)}, }, }, - } + }) validatePodReadiness := func(expectReady bool) { err := wait.Poll(time.Second, time.Minute, func() (bool, error) { @@ -847,20 +847,22 @@ var _ = SIGDescribe("Pods", func() { ginkgo.By("Create set of pods") // create a set of pods in test namespace for _, podTestName := range podTestNames { - _, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podTestName, - Labels: map[string]string{ - "type": "Testing"}, - }, - Spec: v1.PodSpec{ - TerminationGracePeriodSeconds: &one, - Containers: []v1.Container{{ - Image: imageutils.GetE2EImage(imageutils.Agnhost), - Name: "token-test", - }}, - RestartPolicy: v1.RestartPolicyNever, - }}, metav1.CreateOptions{}) + _, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), + e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podTestName, + Labels: map[string]string{ + "type": "Testing", + }, + }, + Spec: v1.PodSpec{ + TerminationGracePeriodSeconds: &one, + Containers: []v1.Container{{ + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Name: "token-test", + }}, + RestartPolicy: v1.RestartPolicyNever, + }}), metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create pod") framework.Logf("created %v", podTestName) } @@ -907,7 +909,7 @@ var _ = SIGDescribe("Pods", func() { podsList, err := f.ClientSet.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{LabelSelector: testPodLabelsFlat}) framework.ExpectNoError(err, "failed to list Pods") - testPod := v1.Pod{ + testPod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: testPodName, Labels: testPodLabels, @@ -921,9 +923,9 @@ var _ = SIGDescribe("Pods", func() { }, }, }, - } + }) ginkgo.By("creating a Pod with a static label") - _, err = f.ClientSet.CoreV1().Pods(testNamespaceName).Create(context.TODO(), &testPod, metav1.CreateOptions{}) + _, err = f.ClientSet.CoreV1().Pods(testNamespaceName).Create(context.TODO(), testPod, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create Pod %v in namespace %v", testPod.ObjectMeta.Name, testNamespaceName) ginkgo.By("watching for Pod to be ready") diff --git a/test/e2e/framework/pod/utils.go b/test/e2e/framework/pod/utils.go index d6a88bf2f4f..66487a92e9d 100644 --- a/test/e2e/framework/pod/utils.go +++ b/test/e2e/framework/pod/utils.go @@ -18,9 +18,14 @@ package pod import ( "flag" + "fmt" + + "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" imageutils "k8s.io/kubernetes/test/utils/image" + psaapi "k8s.io/pod-security-admission/api" + psapolicy "k8s.io/pod-security-admission/policy" "k8s.io/utils/pointer" ) @@ -115,10 +120,16 @@ func GetLinuxLabel() *v1.SELinuxOptions { Level: "s0:c0,c1"} } -// GetRestrictedPodSecurityContext returns a minimal restricted pod security context. +// DefaultNonRootUser is the default user ID used for running restricted (non-root) containers. +const DefaultNonRootUser = 1000 + +// GetRestrictedPodSecurityContext returns a restricted pod security context. +// This includes setting RunAsUser for convenience, to pass the RunAsNonRoot check. +// Tests that require a specific user ID should override this. func GetRestrictedPodSecurityContext() *v1.PodSecurityContext { return &v1.PodSecurityContext{ RunAsNonRoot: pointer.BoolPtr(true), + RunAsUser: pointer.Int64(DefaultNonRootUser), SeccompProfile: &v1.SeccompProfile{Type: v1.SeccompProfileTypeRuntimeDefault}, } } @@ -130,3 +141,69 @@ func GetRestrictedContainerSecurityContext() *v1.SecurityContext { Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, } } + +var psaEvaluator, _ = psapolicy.NewEvaluator(psapolicy.DefaultChecks()) + +// MustMixinRestrictedPodSecurity makes the given pod compliant with the restricted pod security level. +// If doing so would overwrite existing non-conformant configuration, a test failure is triggered. +func MustMixinRestrictedPodSecurity(pod *v1.Pod) *v1.Pod { + err := MixinRestrictedPodSecurity(pod) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + return pod +} + +// MixinRestrictedPodSecurity makes the given pod compliant with the restricted pod security level. +// If doing so would overwrite existing non-conformant configuration, an error is returned. +// Note that this sets a default RunAsUser. See GetRestrictedPodSecurityContext. +// TODO(#105919): Handle PodOS for windows pods. +func MixinRestrictedPodSecurity(pod *v1.Pod) error { + if pod.Spec.SecurityContext == nil { + pod.Spec.SecurityContext = GetRestrictedPodSecurityContext() + } else { + if pod.Spec.SecurityContext.RunAsNonRoot == nil { + pod.Spec.SecurityContext.RunAsNonRoot = pointer.BoolPtr(true) + } + if pod.Spec.SecurityContext.RunAsUser == nil { + pod.Spec.SecurityContext.RunAsUser = pointer.Int64Ptr(DefaultNonRootUser) + } + if pod.Spec.SecurityContext.SeccompProfile == nil { + pod.Spec.SecurityContext.SeccompProfile = &v1.SeccompProfile{Type: v1.SeccompProfileTypeRuntimeDefault} + } + } + for i := range pod.Spec.Containers { + mixinRestrictedContainerSecurityContext(&pod.Spec.Containers[i]) + } + for i := range pod.Spec.InitContainers { + mixinRestrictedContainerSecurityContext(&pod.Spec.InitContainers[i]) + } + + // Validate the resulting pod against the restricted profile. + restricted := psaapi.LevelVersion{ + Level: psaapi.LevelRestricted, + Version: psaapi.LatestVersion(), + } + if agg := psapolicy.AggregateCheckResults(psaEvaluator.EvaluatePod(restricted, &pod.ObjectMeta, &pod.Spec)); !agg.Allowed { + return fmt.Errorf("failed to make pod %s restricted: %s", pod.Name, agg.ForbiddenDetail()) + } + + return nil +} + +// mixinRestrictedContainerSecurityContext adds the required container security context options to +// be compliant with the restricted pod security level. Non-conformance checking is handled by the +// caller. +func mixinRestrictedContainerSecurityContext(container *v1.Container) { + if container.SecurityContext == nil { + container.SecurityContext = GetRestrictedContainerSecurityContext() + } else { + if container.SecurityContext.AllowPrivilegeEscalation == nil { + container.SecurityContext.AllowPrivilegeEscalation = pointer.Bool(false) + } + if container.SecurityContext.Capabilities == nil { + container.SecurityContext.Capabilities = &v1.Capabilities{} + } + if len(container.SecurityContext.Capabilities.Drop) == 0 { + container.SecurityContext.Capabilities.Drop = []v1.Capability{"ALL"} + } + } +} diff --git a/test/e2e/framework/pod/utils_test.go b/test/e2e/framework/pod/utils_test.go new file mode 100644 index 00000000000..cc93ee0bae5 --- /dev/null +++ b/test/e2e/framework/pod/utils_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +func TestMixinRestrictedPodSecurity(t *testing.T) { + restrictablePods := []v1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: "pause", + Image: "pause", + }}, + }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "already_restricted", + }, + Spec: v1.PodSpec{ + SecurityContext: GetRestrictedPodSecurityContext(), + Containers: []v1.Container{{ + Name: "pause", + Image: "pause", + SecurityContext: GetRestrictedContainerSecurityContext(), + }}, + }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "empty_securityContext", + }, + Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{}, + Containers: []v1.Container{{ + Name: "pause", + Image: "pause", + SecurityContext: &v1.SecurityContext{}, + }}, + }, + }} + + for _, pod := range restrictablePods { + t.Run(pod.Name, func(t *testing.T) { + p := pod // closure + assert.NoError(t, MixinRestrictedPodSecurity(&p)) + assert.Equal(t, GetRestrictedPodSecurityContext(), p.Spec.SecurityContext, + "Mixed in PodSecurityContext should equal the from-scratch PodSecurityContext") + assert.Equal(t, GetRestrictedContainerSecurityContext(), p.Spec.Containers[0].SecurityContext, + "Mixed in SecurityContext should equal the from-scratch SecurityContext") + }) + } + + privilegedPod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "privileged", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: "pause", + Image: "pause", + SecurityContext: &v1.SecurityContext{ + Privileged: pointer.Bool(true), + }, + }}, + }, + } + t.Run("privileged", func(t *testing.T) { + assert.Error(t, MixinRestrictedPodSecurity(&privilegedPod)) + }) + +}