Merge pull request #115219 from bobbypage/gh-108804-skipper

e2e node: Update runtime class handler skip logic

test/e2e/common/node/runtimeclass.go

@@ -35,7 +35,7 @@ import (
 	runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
-	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+	e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	admissionapi "k8s.io/pod-security-admission/api"
@@ -45,7 +45,7 @@ import (
 
 var _ = SIGDescribe("RuntimeClass", func() {
 	f := framework.NewDefaultFramework("runtimeclass")
-	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
 
 	/*
 		Release: v1.20
@@ -54,7 +54,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
 	*/
 	framework.ConformanceIt("should reject a Pod requesting a non-existent RuntimeClass [NodeConformance]", func(ctx context.Context) {
 		rcName := f.Namespace.Name + "-nonexistent"
-		expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName))
+		expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
 	})
 
 	// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler not being installed.
@@ -62,7 +62,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
 		handler := f.Namespace.Name + "-handler"
 		rcName := createRuntimeClass(ctx, f, "unconfigured-handler", handler, nil)
 		ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
-		pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
+		pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
 		eventSelector := fields.Set{
 			"involvedObject.kind":      "Pod",
 			"involvedObject.name":      pod.Name,
@@ -83,13 +83,13 @@ var _ = SIGDescribe("RuntimeClass", func() {
 	// This test requires that the PreconfiguredRuntimeClassHandler has already been set up on nodes.
 	// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler installed and working.
 	ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func(ctx context.Context) {
-		// Requires special setup of test-handler which is only done in GCE kube-up environment
+		if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
-		// see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
+			e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
-		e2eskipper.SkipUnlessProviderIs("gce")
+		}
 
-		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil)
+		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
 		ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
-		pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
+		pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
 		expectPodSuccess(ctx, f, pod)
 	})
 
@@ -102,9 +102,9 @@ var _ = SIGDescribe("RuntimeClass", func() {
 		is not being tested here.
 	*/
 	framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass without PodOverhead [NodeConformance]", func(ctx context.Context) {
-		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil)
+		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
 		ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
-		pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
+		pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
 		// there is only one pod in the namespace
 		label := labels.SelectorFromSet(labels.Set(map[string]string{}))
 		pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
@@ -127,14 +127,14 @@ var _ = SIGDescribe("RuntimeClass", func() {
 		is not being tested here.
 	*/
 	framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass and initialize its Overhead [NodeConformance]", func(ctx context.Context) {
-		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
+		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
 			PodFixed: v1.ResourceList{
 				v1.ResourceName(v1.ResourceCPU):    resource.MustParse("10m"),
 				v1.ResourceName(v1.ResourceMemory): resource.MustParse("1Mi"),
 			},
 		})
 		ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
-		pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
+		pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
 		// there is only one pod in the namespace
 		label := labels.SelectorFromSet(labels.Set(map[string]string{}))
 		pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
@@ -174,7 +174,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
 			}))
 		})
 
-		expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName))
+		expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
 	})
 
 	/*

test/e2e/framework/node/runtimeclass.go

@@ -1,51 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package node
-
-import (
-	"fmt"
-
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	imageutils "k8s.io/kubernetes/test/utils/image"
-	utilpointer "k8s.io/utils/pointer"
-)
-
-const (
-	// PreconfiguredRuntimeClassHandler is the name of the runtime handler
-	// that is expected to be preconfigured in the test environment.
-	PreconfiguredRuntimeClassHandler = "test-handler"
-)
-
-// NewRuntimeClassPod returns a test pod with the given runtimeClassName
-func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
-	return &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
-		},
-		Spec: v1.PodSpec{
-			RuntimeClassName: &runtimeClassName,
-			Containers: []v1.Container{{
-				Name:    "test",
-				Image:   imageutils.GetE2EImage(imageutils.BusyBox),
-				Command: []string{"true"},
-			}},
-			RestartPolicy:                v1.RestartPolicyNever,
-			AutomountServiceAccountToken: utilpointer.BoolPtr(false),
-		},
-	}
-}

test/e2e/framework/node/runtimeclass/runtimeclass.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package runtimeclass
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/onsi/ginkgo/v2"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+	storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+	utilpointer "k8s.io/utils/pointer"
+)
+
+const (
+	// PreconfiguredRuntimeClassHandler is the name of the runtime handler
+	// that is expected to be preconfigured in the test environment.
+	PreconfiguredRuntimeClassHandler = "test-handler"
+)
+
+// NewRuntimeClassPod returns a test pod with the given runtimeClassName
+func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
+		},
+		Spec: v1.PodSpec{
+			RuntimeClassName: &runtimeClassName,
+			Containers: []v1.Container{{
+				Name:    "test",
+				Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+				Command: []string{"true"},
+			}},
+			RestartPolicy:                v1.RestartPolicyNever,
+			AutomountServiceAccountToken: utilpointer.BoolPtr(false),
+		},
+	}
+}
+
+// Check if test-handler is configured by reading the configuration from container runtime config.
+// If no error is returned, the container runtime is assumed to support the test-handler, otherwise an error will be returned.
+func NodeSupportsPreconfiguredRuntimeClassHandler(ctx context.Context, f *framework.Framework) error {
+	node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
+	framework.ExpectNoError(err)
+	hostExec := storageutils.NewHostExec(f)
+	ginkgo.DeferCleanup(hostExec.Cleanup)
+
+	// This is a hacky check that greps the container runtime config to determine if the test-handler is the underlying container runtime config.
+	// For containerd, this is configured in kube-up for GCE clusters here: https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
+	// For cri-o, see configuration here: https://github.com/cri-o/cri-o/blob/5dc6a035c940f5dde3a727b2e2d8d4b7e371ad55/contrib/test/ci/e2e-base.yml#L2-L13
+	// If the `runtimes.test-handler` substring is found in the runtime config, it is assumed that the handler is configured.
+	cmd := fmt.Sprintf(`if [ -e '/etc/containerd/config.toml' ]; then
+grep -q 'runtimes.%s' /etc/containerd/config.toml
+  return
+fi
+
+if [ -e '/etc/crio/crio.conf' ]; then
+  grep -q 'runtimes.%s' /etc/crio/crio.conf
+  return
+fi
+`, PreconfiguredRuntimeClassHandler, PreconfiguredRuntimeClassHandler)
+
+	_, err = hostExec.IssueCommandWithResult(ctx, cmd, node)
+	return err
+}

test/e2e/node/runtimeclass.go

@@ -30,6 +30,7 @@ import (
 	runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+	e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	"k8s.io/kubernetes/test/e2e/scheduling"
@@ -40,7 +41,7 @@ import (
 
 var _ = SIGDescribe("RuntimeClass", func() {
 	f := framework.NewDefaultFramework("runtimeclass")
-	f.NamespacePodSecurityEnforceLevel = api.LevelBaseline
+	f.NamespacePodSecurityEnforceLevel = api.LevelPrivileged
 
 	ginkgo.It("should reject a Pod requesting a RuntimeClass with conflicting node selector", func(ctx context.Context) {
 		labelFooName := "foo-" + string(uuid.NewUUID())
@@ -56,7 +57,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
 		rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
 		framework.ExpectNoError(err, "failed to create RuntimeClass resource")
 
-		pod := e2enode.NewRuntimeClassPod(rc.GetName())
+		pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
 		pod.Spec.NodeSelector = map[string]string{
 			labelFooName: "bar",
 		}
@@ -111,7 +112,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
 		rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
 		framework.ExpectNoError(err, "failed to create RuntimeClass resource")
 
-		pod := e2enode.NewRuntimeClassPod(rc.GetName())
+		pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
 		pod.Spec.NodeSelector = map[string]string{
 			labelFooName: "bar",
 		}
@@ -128,9 +129,9 @@ var _ = SIGDescribe("RuntimeClass", func() {
 	})
 
 	ginkgo.It("should run a Pod requesting a RuntimeClass with scheduling without taints ", func(ctx context.Context) {
-		// Requires special setup of test-handler which is only done in GCE kube-up environment
+		if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
-		// see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
+			e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
-		e2eskipper.SkipUnlessProviderIs("gce")
+		}
 
 		labelFooName := "foo-" + string(uuid.NewUUID())
 		labelFizzName := "fizz-" + string(uuid.NewUUID())
@@ -157,7 +158,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
 		rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
 		framework.ExpectNoError(err, "failed to create RuntimeClass resource")
 
-		pod := e2enode.NewRuntimeClassPod(rc.GetName())
+		pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
 		pod.Spec.NodeSelector = map[string]string{
 			labelFooName: "bar",
 		}
@@ -176,5 +177,5 @@ var _ = SIGDescribe("RuntimeClass", func() {
 // newRuntimeClass returns a test runtime class.
 func newRuntimeClass(namespace, name string) *nodev1.RuntimeClass {
 	uniqueName := fmt.Sprintf("%s-%s", namespace, name)
-	return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler)
+	return runtimeclasstest.NewRuntimeClass(uniqueName, e2eruntimeclass.PreconfiguredRuntimeClassHandler)
 }

test/e2e/scheduling/predicates.go

@@ -36,6 +36,7 @@ import (
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+	e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
@@ -238,7 +239,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
 			framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName)
 
 			// Register a runtimeClass with overhead set as 25% of the available beard-seconds
-			handler = e2enode.PreconfiguredRuntimeClassHandler
+			handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler
 
 			rc := &nodev1.RuntimeClass{
 				ObjectMeta: metav1.ObjectMeta{Name: handler},
@@ -270,7 +271,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
 			}
 
 			// remove RuntimeClass
-			_ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2enode.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{})
+			_ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2eruntimeclass.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{})
 		})
 
 		ginkgo.It("verify pod overhead is accounted for", func(ctx context.Context) {

test/e2e/storage/utils/host_exec.go

@@ -76,11 +76,18 @@ func (h *hostExecutor) launchNodeExecPod(ctx context.Context, node string) *v1.P
 
 	hostExecPod := e2epod.NewExecPodSpec(ns.Name, "", true)
 	hostExecPod.GenerateName = fmt.Sprintf("hostexec-%s-", node)
+
+	if framework.TestContext.NodeE2E {
+		// E2E node tests do not run a scheduler, so set the node name directly
+		hostExecPod.Spec.NodeName = node
+	} else {
 		// Use NodeAffinity instead of NodeName so that pods will not
 		// be immediately Failed by kubelet if it's out of space. Instead
 		// Pods will be pending in the scheduler until there is space freed
 		// up.
 		e2epod.SetNodeAffinity(&hostExecPod.Spec, node)
+
+	}
 	hostExecPod.Spec.Volumes = []v1.Volume{
 		{
 			// Required to enter into host mount namespace via nsenter.

test/e2e_node/runtimeclass_test.go

@@ -26,7 +26,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	"k8s.io/kubernetes/test/e2e/framework"
-	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+	e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 	admissionapi "k8s.io/pod-security-admission/api"
@@ -106,7 +106,7 @@ var _ = SIGDescribe("Kubelet PodOverhead handling [LinuxOnly]", func() {
 					handler       string
 				)
 				ginkgo.By("Creating a RuntimeClass with Overhead definied", func() {
-					handler = e2enode.PreconfiguredRuntimeClassHandler
+					handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler
 					rc := &nodev1.RuntimeClass{
 						ObjectMeta: metav1.ObjectMeta{Name: handler},
 						Handler:    handler,