diff --git a/test/e2e/common/node/runtimeclass.go b/test/e2e/common/node/runtimeclass.go index 673b73d5130..11e8faab8e1 100644 --- a/test/e2e/common/node/runtimeclass.go +++ b/test/e2e/common/node/runtimeclass.go @@ -35,7 +35,7 @@ import ( runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" "k8s.io/kubernetes/test/e2e/framework" e2eevents "k8s.io/kubernetes/test/e2e/framework/events" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" admissionapi "k8s.io/pod-security-admission/api" @@ -45,7 +45,7 @@ import ( var _ = SIGDescribe("RuntimeClass", func() { f := framework.NewDefaultFramework("runtimeclass") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged /* Release: v1.20 @@ -54,7 +54,7 @@ var _ = SIGDescribe("RuntimeClass", func() { */ framework.ConformanceIt("should reject a Pod requesting a non-existent RuntimeClass [NodeConformance]", func(ctx context.Context) { rcName := f.Namespace.Name + "-nonexistent" - expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName)) + expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName)) }) // The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler not being installed. @@ -62,7 +62,7 @@ var _ = SIGDescribe("RuntimeClass", func() { handler := f.Namespace.Name + "-handler" rcName := createRuntimeClass(ctx, f, "unconfigured-handler", handler, nil) ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName) - pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName)) + pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName)) eventSelector := fields.Set{ "involvedObject.kind": "Pod", "involvedObject.name": pod.Name, @@ -83,13 +83,13 @@ var _ = SIGDescribe("RuntimeClass", func() { // This test requires that the PreconfiguredRuntimeClassHandler has already been set up on nodes. // The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler installed and working. ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func(ctx context.Context) { - // Requires special setup of test-handler which is only done in GCE kube-up environment - // see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076 - e2eskipper.SkipUnlessProviderIs("gce") + if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil { + e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err) + } - rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil) + rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil) ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName) - pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName)) + pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName)) expectPodSuccess(ctx, f, pod) }) @@ -102,9 +102,9 @@ var _ = SIGDescribe("RuntimeClass", func() { is not being tested here. */ framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass without PodOverhead [NodeConformance]", func(ctx context.Context) { - rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil) + rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil) ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName) - pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName)) + pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName)) // there is only one pod in the namespace label := labels.SelectorFromSet(labels.Set(map[string]string{})) pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label) @@ -127,14 +127,14 @@ var _ = SIGDescribe("RuntimeClass", func() { is not being tested here. */ framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass and initialize its Overhead [NodeConformance]", func(ctx context.Context) { - rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{ + rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{ PodFixed: v1.ResourceList{ v1.ResourceName(v1.ResourceCPU): resource.MustParse("10m"), v1.ResourceName(v1.ResourceMemory): resource.MustParse("1Mi"), }, }) ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName) - pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName)) + pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName)) // there is only one pod in the namespace label := labels.SelectorFromSet(labels.Set(map[string]string{})) pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label) @@ -174,7 +174,7 @@ var _ = SIGDescribe("RuntimeClass", func() { })) }) - expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName)) + expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName)) }) /* diff --git a/test/e2e/framework/node/runtimeclass.go b/test/e2e/framework/node/runtimeclass.go deleted file mode 100644 index a813233edb1..00000000000 --- a/test/e2e/framework/node/runtimeclass.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - imageutils "k8s.io/kubernetes/test/utils/image" - utilpointer "k8s.io/utils/pointer" -) - -const ( - // PreconfiguredRuntimeClassHandler is the name of the runtime handler - // that is expected to be preconfigured in the test environment. - PreconfiguredRuntimeClassHandler = "test-handler" -) - -// NewRuntimeClassPod returns a test pod with the given runtimeClassName -func NewRuntimeClassPod(runtimeClassName string) *v1.Pod { - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName), - }, - Spec: v1.PodSpec{ - RuntimeClassName: &runtimeClassName, - Containers: []v1.Container{{ - Name: "test", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"true"}, - }}, - RestartPolicy: v1.RestartPolicyNever, - AutomountServiceAccountToken: utilpointer.BoolPtr(false), - }, - } -} diff --git a/test/e2e/framework/node/runtimeclass/runtimeclass.go b/test/e2e/framework/node/runtimeclass/runtimeclass.go new file mode 100644 index 00000000000..8ec8f7ae95e --- /dev/null +++ b/test/e2e/framework/node/runtimeclass/runtimeclass.go @@ -0,0 +1,83 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtimeclass + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + storageutils "k8s.io/kubernetes/test/e2e/storage/utils" + imageutils "k8s.io/kubernetes/test/utils/image" + utilpointer "k8s.io/utils/pointer" +) + +const ( + // PreconfiguredRuntimeClassHandler is the name of the runtime handler + // that is expected to be preconfigured in the test environment. + PreconfiguredRuntimeClassHandler = "test-handler" +) + +// NewRuntimeClassPod returns a test pod with the given runtimeClassName +func NewRuntimeClassPod(runtimeClassName string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName), + }, + Spec: v1.PodSpec{ + RuntimeClassName: &runtimeClassName, + Containers: []v1.Container{{ + Name: "test", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"true"}, + }}, + RestartPolicy: v1.RestartPolicyNever, + AutomountServiceAccountToken: utilpointer.BoolPtr(false), + }, + } +} + +// Check if test-handler is configured by reading the configuration from container runtime config. +// If no error is returned, the container runtime is assumed to support the test-handler, otherwise an error will be returned. +func NodeSupportsPreconfiguredRuntimeClassHandler(ctx context.Context, f *framework.Framework) error { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + hostExec := storageutils.NewHostExec(f) + ginkgo.DeferCleanup(hostExec.Cleanup) + + // This is a hacky check that greps the container runtime config to determine if the test-handler is the underlying container runtime config. + // For containerd, this is configured in kube-up for GCE clusters here: https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076 + // For cri-o, see configuration here: https://github.com/cri-o/cri-o/blob/5dc6a035c940f5dde3a727b2e2d8d4b7e371ad55/contrib/test/ci/e2e-base.yml#L2-L13 + // If the `runtimes.test-handler` substring is found in the runtime config, it is assumed that the handler is configured. + cmd := fmt.Sprintf(`if [ -e '/etc/containerd/config.toml' ]; then +grep -q 'runtimes.%s' /etc/containerd/config.toml + return +fi + +if [ -e '/etc/crio/crio.conf' ]; then + grep -q 'runtimes.%s' /etc/crio/crio.conf + return +fi +`, PreconfiguredRuntimeClassHandler, PreconfiguredRuntimeClassHandler) + + _, err = hostExec.IssueCommandWithResult(ctx, cmd, node) + return err +} diff --git a/test/e2e/node/runtimeclass.go b/test/e2e/node/runtimeclass.go index c46a7329584..ac6ef1ceb64 100644 --- a/test/e2e/node/runtimeclass.go +++ b/test/e2e/node/runtimeclass.go @@ -30,6 +30,7 @@ import ( runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/scheduling" @@ -40,7 +41,7 @@ import ( var _ = SIGDescribe("RuntimeClass", func() { f := framework.NewDefaultFramework("runtimeclass") - f.NamespacePodSecurityEnforceLevel = api.LevelBaseline + f.NamespacePodSecurityEnforceLevel = api.LevelPrivileged ginkgo.It("should reject a Pod requesting a RuntimeClass with conflicting node selector", func(ctx context.Context) { labelFooName := "foo-" + string(uuid.NewUUID()) @@ -56,7 +57,7 @@ var _ = SIGDescribe("RuntimeClass", func() { rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create RuntimeClass resource") - pod := e2enode.NewRuntimeClassPod(rc.GetName()) + pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName()) pod.Spec.NodeSelector = map[string]string{ labelFooName: "bar", } @@ -111,7 +112,7 @@ var _ = SIGDescribe("RuntimeClass", func() { rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create RuntimeClass resource") - pod := e2enode.NewRuntimeClassPod(rc.GetName()) + pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName()) pod.Spec.NodeSelector = map[string]string{ labelFooName: "bar", } @@ -128,9 +129,9 @@ var _ = SIGDescribe("RuntimeClass", func() { }) ginkgo.It("should run a Pod requesting a RuntimeClass with scheduling without taints ", func(ctx context.Context) { - // Requires special setup of test-handler which is only done in GCE kube-up environment - // see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076 - e2eskipper.SkipUnlessProviderIs("gce") + if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil { + e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err) + } labelFooName := "foo-" + string(uuid.NewUUID()) labelFizzName := "fizz-" + string(uuid.NewUUID()) @@ -157,7 +158,7 @@ var _ = SIGDescribe("RuntimeClass", func() { rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create RuntimeClass resource") - pod := e2enode.NewRuntimeClassPod(rc.GetName()) + pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName()) pod.Spec.NodeSelector = map[string]string{ labelFooName: "bar", } @@ -176,5 +177,5 @@ var _ = SIGDescribe("RuntimeClass", func() { // newRuntimeClass returns a test runtime class. func newRuntimeClass(namespace, name string) *nodev1.RuntimeClass { uniqueName := fmt.Sprintf("%s-%s", namespace, name) - return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler) + return runtimeclasstest.NewRuntimeClass(uniqueName, e2eruntimeclass.PreconfiguredRuntimeClassHandler) } diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index 0079a576ebf..6967961a143 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -36,6 +36,7 @@ import ( podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2erc "k8s.io/kubernetes/test/e2e/framework/rc" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" @@ -238,7 +239,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName) // Register a runtimeClass with overhead set as 25% of the available beard-seconds - handler = e2enode.PreconfiguredRuntimeClassHandler + handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler rc := &nodev1.RuntimeClass{ ObjectMeta: metav1.ObjectMeta{Name: handler}, @@ -270,7 +271,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { } // remove RuntimeClass - _ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2enode.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{}) + _ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2eruntimeclass.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{}) }) ginkgo.It("verify pod overhead is accounted for", func(ctx context.Context) { diff --git a/test/e2e_node/runtimeclass_test.go b/test/e2e_node/runtimeclass_test.go index 1dca583b1cf..2103fecbb85 100644 --- a/test/e2e_node/runtimeclass_test.go +++ b/test/e2e_node/runtimeclass_test.go @@ -26,7 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/framework" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -106,7 +106,7 @@ var _ = SIGDescribe("Kubelet PodOverhead handling [LinuxOnly]", func() { handler string ) ginkgo.By("Creating a RuntimeClass with Overhead definied", func() { - handler = e2enode.PreconfiguredRuntimeClassHandler + handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler rc := &nodev1.RuntimeClass{ ObjectMeta: metav1.ObjectMeta{Name: handler}, Handler: handler,