From b96290c08f38524312c33ddb993470ff0ddb40f1 Mon Sep 17 00:00:00 2001 From: David Porter Date: Wed, 18 Jan 2023 18:37:12 -0800 Subject: [PATCH] e2e node: Update runtime class handler skip logic There are two runtime class tests which required the container runtime config to include explicit configuration for `test-handler`. The current logic skips these tests in non GCE environments. This skip is too strict since the test is skipped in node e2e environments and in other environments such as kind, which support running the test and also configure `test-handler`. Instead of skipping based on provider, add a new function `NodeSupportsPreconfiguredRuntimeClassHandler` which examines the underlying container runtime config and checks if the config includes `test-handler`. The check is a bit brittle since it assumes container runtime config paths, but it is a net improvement over skipping the test entirely on non GCE environments. This results in the test working in the common test environments, namely GCE kube-up, node e2e, and kind. Signed-off-by: David Porter --- test/e2e/common/node/runtimeclass.go | 28 +++---- test/e2e/framework/node/runtimeclass.go | 51 ------------ .../node/runtimeclass/runtimeclass.go | 83 +++++++++++++++++++ test/e2e/node/runtimeclass.go | 17 ++-- test/e2e/scheduling/predicates.go | 5 +- test/e2e_node/runtimeclass_test.go | 4 +- 6 files changed, 111 insertions(+), 77 deletions(-) delete mode 100644 test/e2e/framework/node/runtimeclass.go create mode 100644 test/e2e/framework/node/runtimeclass/runtimeclass.go diff --git a/test/e2e/common/node/runtimeclass.go b/test/e2e/common/node/runtimeclass.go index 673b73d5130..11e8faab8e1 100644 --- a/test/e2e/common/node/runtimeclass.go +++ b/test/e2e/common/node/runtimeclass.go @@ -35,7 +35,7 @@ import ( runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" "k8s.io/kubernetes/test/e2e/framework" e2eevents "k8s.io/kubernetes/test/e2e/framework/events" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" admissionapi "k8s.io/pod-security-admission/api" @@ -45,7 +45,7 @@ import ( var _ = SIGDescribe("RuntimeClass", func() { f := framework.NewDefaultFramework("runtimeclass") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged /* Release: v1.20 @@ -54,7 +54,7 @@ var _ = SIGDescribe("RuntimeClass", func() { */ framework.ConformanceIt("should reject a Pod requesting a non-existent RuntimeClass [NodeConformance]", func(ctx context.Context) { rcName := f.Namespace.Name + "-nonexistent" - expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName)) + expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName)) }) // The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler not being installed. @@ -62,7 +62,7 @@ var _ = SIGDescribe("RuntimeClass", func() { handler := f.Namespace.Name + "-handler" rcName := createRuntimeClass(ctx, f, "unconfigured-handler", handler, nil) ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName) - pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName)) + pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName)) eventSelector := fields.Set{ "involvedObject.kind": "Pod", "involvedObject.name": pod.Name, @@ -83,13 +83,13 @@ var _ = SIGDescribe("RuntimeClass", func() { // This test requires that the PreconfiguredRuntimeClassHandler has already been set up on nodes. // The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler installed and working. ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func(ctx context.Context) { - // Requires special setup of test-handler which is only done in GCE kube-up environment - // see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076 - e2eskipper.SkipUnlessProviderIs("gce") + if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil { + e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err) + } - rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil) + rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil) ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName) - pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName)) + pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName)) expectPodSuccess(ctx, f, pod) }) @@ -102,9 +102,9 @@ var _ = SIGDescribe("RuntimeClass", func() { is not being tested here. */ framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass without PodOverhead [NodeConformance]", func(ctx context.Context) { - rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil) + rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil) ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName) - pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName)) + pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName)) // there is only one pod in the namespace label := labels.SelectorFromSet(labels.Set(map[string]string{})) pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label) @@ -127,14 +127,14 @@ var _ = SIGDescribe("RuntimeClass", func() { is not being tested here. */ framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass and initialize its Overhead [NodeConformance]", func(ctx context.Context) { - rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{ + rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{ PodFixed: v1.ResourceList{ v1.ResourceName(v1.ResourceCPU): resource.MustParse("10m"), v1.ResourceName(v1.ResourceMemory): resource.MustParse("1Mi"), }, }) ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName) - pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName)) + pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName)) // there is only one pod in the namespace label := labels.SelectorFromSet(labels.Set(map[string]string{})) pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label) @@ -174,7 +174,7 @@ var _ = SIGDescribe("RuntimeClass", func() { })) }) - expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName)) + expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName)) }) /* diff --git a/test/e2e/framework/node/runtimeclass.go b/test/e2e/framework/node/runtimeclass.go deleted file mode 100644 index a813233edb1..00000000000 --- a/test/e2e/framework/node/runtimeclass.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - imageutils "k8s.io/kubernetes/test/utils/image" - utilpointer "k8s.io/utils/pointer" -) - -const ( - // PreconfiguredRuntimeClassHandler is the name of the runtime handler - // that is expected to be preconfigured in the test environment. - PreconfiguredRuntimeClassHandler = "test-handler" -) - -// NewRuntimeClassPod returns a test pod with the given runtimeClassName -func NewRuntimeClassPod(runtimeClassName string) *v1.Pod { - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName), - }, - Spec: v1.PodSpec{ - RuntimeClassName: &runtimeClassName, - Containers: []v1.Container{{ - Name: "test", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"true"}, - }}, - RestartPolicy: v1.RestartPolicyNever, - AutomountServiceAccountToken: utilpointer.BoolPtr(false), - }, - } -} diff --git a/test/e2e/framework/node/runtimeclass/runtimeclass.go b/test/e2e/framework/node/runtimeclass/runtimeclass.go new file mode 100644 index 00000000000..8ec8f7ae95e --- /dev/null +++ b/test/e2e/framework/node/runtimeclass/runtimeclass.go @@ -0,0 +1,83 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtimeclass + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + storageutils "k8s.io/kubernetes/test/e2e/storage/utils" + imageutils "k8s.io/kubernetes/test/utils/image" + utilpointer "k8s.io/utils/pointer" +) + +const ( + // PreconfiguredRuntimeClassHandler is the name of the runtime handler + // that is expected to be preconfigured in the test environment. + PreconfiguredRuntimeClassHandler = "test-handler" +) + +// NewRuntimeClassPod returns a test pod with the given runtimeClassName +func NewRuntimeClassPod(runtimeClassName string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName), + }, + Spec: v1.PodSpec{ + RuntimeClassName: &runtimeClassName, + Containers: []v1.Container{{ + Name: "test", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"true"}, + }}, + RestartPolicy: v1.RestartPolicyNever, + AutomountServiceAccountToken: utilpointer.BoolPtr(false), + }, + } +} + +// Check if test-handler is configured by reading the configuration from container runtime config. +// If no error is returned, the container runtime is assumed to support the test-handler, otherwise an error will be returned. +func NodeSupportsPreconfiguredRuntimeClassHandler(ctx context.Context, f *framework.Framework) error { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + hostExec := storageutils.NewHostExec(f) + ginkgo.DeferCleanup(hostExec.Cleanup) + + // This is a hacky check that greps the container runtime config to determine if the test-handler is the underlying container runtime config. + // For containerd, this is configured in kube-up for GCE clusters here: https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076 + // For cri-o, see configuration here: https://github.com/cri-o/cri-o/blob/5dc6a035c940f5dde3a727b2e2d8d4b7e371ad55/contrib/test/ci/e2e-base.yml#L2-L13 + // If the `runtimes.test-handler` substring is found in the runtime config, it is assumed that the handler is configured. + cmd := fmt.Sprintf(`if [ -e '/etc/containerd/config.toml' ]; then +grep -q 'runtimes.%s' /etc/containerd/config.toml + return +fi + +if [ -e '/etc/crio/crio.conf' ]; then + grep -q 'runtimes.%s' /etc/crio/crio.conf + return +fi +`, PreconfiguredRuntimeClassHandler, PreconfiguredRuntimeClassHandler) + + _, err = hostExec.IssueCommandWithResult(ctx, cmd, node) + return err +} diff --git a/test/e2e/node/runtimeclass.go b/test/e2e/node/runtimeclass.go index c46a7329584..ac6ef1ceb64 100644 --- a/test/e2e/node/runtimeclass.go +++ b/test/e2e/node/runtimeclass.go @@ -30,6 +30,7 @@ import ( runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/scheduling" @@ -40,7 +41,7 @@ import ( var _ = SIGDescribe("RuntimeClass", func() { f := framework.NewDefaultFramework("runtimeclass") - f.NamespacePodSecurityEnforceLevel = api.LevelBaseline + f.NamespacePodSecurityEnforceLevel = api.LevelPrivileged ginkgo.It("should reject a Pod requesting a RuntimeClass with conflicting node selector", func(ctx context.Context) { labelFooName := "foo-" + string(uuid.NewUUID()) @@ -56,7 +57,7 @@ var _ = SIGDescribe("RuntimeClass", func() { rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create RuntimeClass resource") - pod := e2enode.NewRuntimeClassPod(rc.GetName()) + pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName()) pod.Spec.NodeSelector = map[string]string{ labelFooName: "bar", } @@ -111,7 +112,7 @@ var _ = SIGDescribe("RuntimeClass", func() { rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create RuntimeClass resource") - pod := e2enode.NewRuntimeClassPod(rc.GetName()) + pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName()) pod.Spec.NodeSelector = map[string]string{ labelFooName: "bar", } @@ -128,9 +129,9 @@ var _ = SIGDescribe("RuntimeClass", func() { }) ginkgo.It("should run a Pod requesting a RuntimeClass with scheduling without taints ", func(ctx context.Context) { - // Requires special setup of test-handler which is only done in GCE kube-up environment - // see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076 - e2eskipper.SkipUnlessProviderIs("gce") + if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil { + e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err) + } labelFooName := "foo-" + string(uuid.NewUUID()) labelFizzName := "fizz-" + string(uuid.NewUUID()) @@ -157,7 +158,7 @@ var _ = SIGDescribe("RuntimeClass", func() { rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create RuntimeClass resource") - pod := e2enode.NewRuntimeClassPod(rc.GetName()) + pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName()) pod.Spec.NodeSelector = map[string]string{ labelFooName: "bar", } @@ -176,5 +177,5 @@ var _ = SIGDescribe("RuntimeClass", func() { // newRuntimeClass returns a test runtime class. func newRuntimeClass(namespace, name string) *nodev1.RuntimeClass { uniqueName := fmt.Sprintf("%s-%s", namespace, name) - return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler) + return runtimeclasstest.NewRuntimeClass(uniqueName, e2eruntimeclass.PreconfiguredRuntimeClassHandler) } diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index 0079a576ebf..6967961a143 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -36,6 +36,7 @@ import ( podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2erc "k8s.io/kubernetes/test/e2e/framework/rc" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" @@ -238,7 +239,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName) // Register a runtimeClass with overhead set as 25% of the available beard-seconds - handler = e2enode.PreconfiguredRuntimeClassHandler + handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler rc := &nodev1.RuntimeClass{ ObjectMeta: metav1.ObjectMeta{Name: handler}, @@ -270,7 +271,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { } // remove RuntimeClass - _ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2enode.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{}) + _ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2eruntimeclass.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{}) }) ginkgo.It("verify pod overhead is accounted for", func(ctx context.Context) { diff --git a/test/e2e_node/runtimeclass_test.go b/test/e2e_node/runtimeclass_test.go index 1dca583b1cf..2103fecbb85 100644 --- a/test/e2e_node/runtimeclass_test.go +++ b/test/e2e_node/runtimeclass_test.go @@ -26,7 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/framework" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" @@ -106,7 +106,7 @@ var _ = SIGDescribe("Kubelet PodOverhead handling [LinuxOnly]", func() { handler string ) ginkgo.By("Creating a RuntimeClass with Overhead definied", func() { - handler = e2enode.PreconfiguredRuntimeClassHandler + handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler rc := &nodev1.RuntimeClass{ ObjectMeta: metav1.ObjectMeta{Name: handler}, Handler: handler,