github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-19 01:40:13 +00:00
Code Issues Projects Releases Wiki Activity

e2e node: Update runtime class handler skip logic

Browse Source 
There are two runtime class tests which required the container runtime
config to include explicit configuration for `test-handler`. The current
logic skips these tests in non GCE environments. This skip is too strict
since the test is skipped in node e2e environments and in other
environments such as kind, which support running the test and also
configure `test-handler`.

Instead of skipping based on provider, add a new function
`NodeSupportsPreconfiguredRuntimeClassHandler` which examines the
underlying container runtime config and checks if the config includes
`test-handler`. The check is a bit brittle since it assumes container
runtime config paths, but it is a net improvement over skipping the test
entirely on non GCE environments.

This results in the test working in the common test environments, namely
GCE kube-up, node e2e, and kind.

Signed-off-by: David Porter <david@porter.me>
This commit is contained in:
David Porter 2023-01-18 18:37:12 -08:00
parent c865a641d3
commit b96290c08f
6 changed files with 111 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
test/e2e/common/node/runtimeclass.go
View File

@ -35,7 +35,7 @@ import (
runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
"k8s.io/kubernetes/test/e2e/framework"
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
admissionapi "k8s.io/pod-security-admission/api"
@ -45,7 +45,7 @@ import (
var _ = SIGDescribe("RuntimeClass", func() {
f := framework.NewDefaultFramework("runtimeclass")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
/*
Release: v1.20
@ -54,7 +54,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
*/
framework.ConformanceIt("should reject a Pod requesting a non-existent RuntimeClass [NodeConformance]", func(ctx context.Context) {
rcName := f.Namespace.Name + "-nonexistent"
expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName))
expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
})
// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler not being installed.
@ -62,7 +62,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
handler := f.Namespace.Name + "-handler"
rcName := createRuntimeClass(ctx, f, "unconfigured-handler", handler, nil)
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
eventSelector := fields.Set{
"involvedObject.kind": "Pod",
"involvedObject.name": pod.Name,
@ -83,13 +83,13 @@ var _ = SIGDescribe("RuntimeClass", func() {
// This test requires that the PreconfiguredRuntimeClassHandler has already been set up on nodes.
// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler installed and working.
ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func(ctx context.Context) {
// Requires special setup of test-handler which is only done in GCE kube-up environment
// see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
e2eskipper.SkipUnlessProviderIs("gce")
if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
}
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil)
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
expectPodSuccess(ctx, f, pod)
})
@ -102,9 +102,9 @@ var _ = SIGDescribe("RuntimeClass", func() {
is not being tested here.
*/
framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass without PodOverhead [NodeConformance]", func(ctx context.Context) {
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil)
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
// there is only one pod in the namespace
label := labels.SelectorFromSet(labels.Set(map[string]string{}))
pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
@ -127,14 +127,14 @@ var _ = SIGDescribe("RuntimeClass", func() {
is not being tested here.
*/
framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass and initialize its Overhead [NodeConformance]", func(ctx context.Context) {
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
PodFixed: v1.ResourceList{
v1.ResourceName(v1.ResourceCPU): resource.MustParse("10m"),
v1.ResourceName(v1.ResourceMemory): resource.MustParse("1Mi"),
},
})
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
// there is only one pod in the namespace
label := labels.SelectorFromSet(labels.Set(map[string]string{}))
pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
@ -174,7 +174,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
}))
})
expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName))
expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
})
/*

51
test/e2e/framework/node/runtimeclass.go
View File

@ -1,51 +0,0 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package node
import (
"fmt"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
imageutils "k8s.io/kubernetes/test/utils/image"
utilpointer "k8s.io/utils/pointer"
)
const (
// PreconfiguredRuntimeClassHandler is the name of the runtime handler
// that is expected to be preconfigured in the test environment.
PreconfiguredRuntimeClassHandler = "test-handler"
)
// NewRuntimeClassPod returns a test pod with the given runtimeClassName
func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
},
Spec: v1.PodSpec{
RuntimeClassName: &runtimeClassName,
Containers: []v1.Container{{
Name: "test",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"true"},
}},
RestartPolicy: v1.RestartPolicyNever,
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
},
}
}

83
test/e2e/framework/node/runtimeclass/runtimeclass.go Normal file
View File

@ -0,0 +1,83 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package runtimeclass
import (
"context"
"fmt"
"github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
utilpointer "k8s.io/utils/pointer"
)
const (
// PreconfiguredRuntimeClassHandler is the name of the runtime handler
// that is expected to be preconfigured in the test environment.
PreconfiguredRuntimeClassHandler = "test-handler"
)
// NewRuntimeClassPod returns a test pod with the given runtimeClassName
func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
},
Spec: v1.PodSpec{
RuntimeClassName: &runtimeClassName,
Containers: []v1.Container{{
Name: "test",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"true"},
}},
RestartPolicy: v1.RestartPolicyNever,
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
},
}
}
// Check if test-handler is configured by reading the configuration from container runtime config.
// If no error is returned, the container runtime is assumed to support the test-handler, otherwise an error will be returned.
func NodeSupportsPreconfiguredRuntimeClassHandler(ctx context.Context, f *framework.Framework) error {
node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
framework.ExpectNoError(err)
hostExec := storageutils.NewHostExec(f)
ginkgo.DeferCleanup(hostExec.Cleanup)
// This is a hacky check that greps the container runtime config to determine if the test-handler is the underlying container runtime config.
// For containerd, this is configured in kube-up for GCE clusters here: https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
// For cri-o, see configuration here: https://github.com/cri-o/cri-o/blob/5dc6a035c940f5dde3a727b2e2d8d4b7e371ad55/contrib/test/ci/e2e-base.yml#L2-L13
// If the `runtimes.test-handler` substring is found in the runtime config, it is assumed that the handler is configured.
cmd := fmt.Sprintf(`if [ -e '/etc/containerd/config.toml' ]; then
grep -q 'runtimes.%s' /etc/containerd/config.toml
return
fi
if [ -e '/etc/crio/crio.conf' ]; then
grep -q 'runtimes.%s' /etc/crio/crio.conf
return
fi
`, PreconfiguredRuntimeClassHandler, PreconfiguredRuntimeClassHandler)
_, err = hostExec.IssueCommandWithResult(ctx, cmd, node)
return err
}

17
test/e2e/node/runtimeclass.go
View File

@ -30,6 +30,7 @@ import (
runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
"k8s.io/kubernetes/test/e2e/scheduling"
@ -40,7 +41,7 @@ import (
var _ = SIGDescribe("RuntimeClass", func() {
f := framework.NewDefaultFramework("runtimeclass")
f.NamespacePodSecurityEnforceLevel = api.LevelBaseline
f.NamespacePodSecurityEnforceLevel = api.LevelPrivileged
ginkgo.It("should reject a Pod requesting a RuntimeClass with conflicting node selector", func(ctx context.Context) {
labelFooName := "foo-" + string(uuid.NewUUID())
@ -56,7 +57,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
pod := e2enode.NewRuntimeClassPod(rc.GetName())
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
pod.Spec.NodeSelector = map[string]string{
labelFooName: "bar",
}
@ -111,7 +112,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
pod := e2enode.NewRuntimeClassPod(rc.GetName())
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
pod.Spec.NodeSelector = map[string]string{
labelFooName: "bar",
}
@ -128,9 +129,9 @@ var _ = SIGDescribe("RuntimeClass", func() {
})
ginkgo.It("should run a Pod requesting a RuntimeClass with scheduling without taints ", func(ctx context.Context) {
// Requires special setup of test-handler which is only done in GCE kube-up environment
// see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
e2eskipper.SkipUnlessProviderIs("gce")
if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
}
labelFooName := "foo-" + string(uuid.NewUUID())
labelFizzName := "fizz-" + string(uuid.NewUUID())
@ -157,7 +158,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
pod := e2enode.NewRuntimeClassPod(rc.GetName())
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
pod.Spec.NodeSelector = map[string]string{
labelFooName: "bar",
}
@ -176,5 +177,5 @@ var _ = SIGDescribe("RuntimeClass", func() {
// newRuntimeClass returns a test runtime class.
func newRuntimeClass(namespace, name string) *nodev1.RuntimeClass {
uniqueName := fmt.Sprintf("%s-%s", namespace, name)
return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler)
return runtimeclasstest.NewRuntimeClass(uniqueName, e2eruntimeclass.PreconfiguredRuntimeClassHandler)
}

5
test/e2e/scheduling/predicates.go
View File

@ -36,6 +36,7 @@ import (
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
@ -238,7 +239,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName)
// Register a runtimeClass with overhead set as 25% of the available beard-seconds
handler = e2enode.PreconfiguredRuntimeClassHandler
handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler
rc := &nodev1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: handler},
@ -270,7 +271,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
}
// remove RuntimeClass
_ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2enode.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{})
_ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2eruntimeclass.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{})
})
ginkgo.It("verify pod overhead is accounted for", func(ctx context.Context) {

4
test/e2e_node/runtimeclass_test.go
View File

@ -26,7 +26,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -106,7 +106,7 @@ var _ = SIGDescribe("Kubelet PodOverhead handling [LinuxOnly]", func() {
handler string
)
ginkgo.By("Creating a RuntimeClass with Overhead definied", func() {
handler = e2enode.PreconfiguredRuntimeClassHandler
handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler
rc := &nodev1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: handler},
Handler: handler,