Merge pull request #115219 from bobbypage/gh-108804-skipper

e2e node: Update runtime class handler skip logic
This commit is contained in:
Kubernetes Prow Robot 2023-01-26 17:42:25 -08:00 committed by GitHub
commit cec91d9c00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 123 additions and 82 deletions

View File

@ -35,7 +35,7 @@ import (
runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
"k8s.io/kubernetes/test/e2e/framework"
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
admissionapi "k8s.io/pod-security-admission/api"
@ -45,7 +45,7 @@ import (
var _ = SIGDescribe("RuntimeClass", func() {
f := framework.NewDefaultFramework("runtimeclass")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
/*
Release: v1.20
@ -54,7 +54,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
*/
framework.ConformanceIt("should reject a Pod requesting a non-existent RuntimeClass [NodeConformance]", func(ctx context.Context) {
rcName := f.Namespace.Name + "-nonexistent"
expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName))
expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
})
// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler not being installed.
@ -62,7 +62,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
handler := f.Namespace.Name + "-handler"
rcName := createRuntimeClass(ctx, f, "unconfigured-handler", handler, nil)
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
eventSelector := fields.Set{
"involvedObject.kind": "Pod",
"involvedObject.name": pod.Name,
@ -83,13 +83,13 @@ var _ = SIGDescribe("RuntimeClass", func() {
// This test requires that the PreconfiguredRuntimeClassHandler has already been set up on nodes.
// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler installed and working.
ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func(ctx context.Context) {
// Requires special setup of test-handler which is only done in GCE kube-up environment
// see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
e2eskipper.SkipUnlessProviderIs("gce")
if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
}
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil)
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
expectPodSuccess(ctx, f, pod)
})
@ -102,9 +102,9 @@ var _ = SIGDescribe("RuntimeClass", func() {
is not being tested here.
*/
framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass without PodOverhead [NodeConformance]", func(ctx context.Context) {
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil)
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
// there is only one pod in the namespace
label := labels.SelectorFromSet(labels.Set(map[string]string{}))
pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
@ -127,14 +127,14 @@ var _ = SIGDescribe("RuntimeClass", func() {
is not being tested here.
*/
framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass and initialize its Overhead [NodeConformance]", func(ctx context.Context) {
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
PodFixed: v1.ResourceList{
v1.ResourceName(v1.ResourceCPU): resource.MustParse("10m"),
v1.ResourceName(v1.ResourceMemory): resource.MustParse("1Mi"),
},
})
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
// there is only one pod in the namespace
label := labels.SelectorFromSet(labels.Set(map[string]string{}))
pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
@ -174,7 +174,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
}))
})
expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName))
expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
})
/*

View File

@ -1,51 +0,0 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package node
import (
"fmt"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
imageutils "k8s.io/kubernetes/test/utils/image"
utilpointer "k8s.io/utils/pointer"
)
const (
// PreconfiguredRuntimeClassHandler is the name of the runtime handler
// that is expected to be preconfigured in the test environment.
PreconfiguredRuntimeClassHandler = "test-handler"
)
// NewRuntimeClassPod returns a test pod with the given runtimeClassName
func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
},
Spec: v1.PodSpec{
RuntimeClassName: &runtimeClassName,
Containers: []v1.Container{{
Name: "test",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"true"},
}},
RestartPolicy: v1.RestartPolicyNever,
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
},
}
}

View File

@ -0,0 +1,83 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package runtimeclass
import (
"context"
"fmt"
"github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
utilpointer "k8s.io/utils/pointer"
)
const (
// PreconfiguredRuntimeClassHandler is the name of the runtime handler
// that is expected to be preconfigured in the test environment.
PreconfiguredRuntimeClassHandler = "test-handler"
)
// NewRuntimeClassPod returns a test pod with the given runtimeClassName
func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
},
Spec: v1.PodSpec{
RuntimeClassName: &runtimeClassName,
Containers: []v1.Container{{
Name: "test",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"true"},
}},
RestartPolicy: v1.RestartPolicyNever,
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
},
}
}
// Check if test-handler is configured by reading the configuration from container runtime config.
// If no error is returned, the container runtime is assumed to support the test-handler, otherwise an error will be returned.
func NodeSupportsPreconfiguredRuntimeClassHandler(ctx context.Context, f *framework.Framework) error {
node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
framework.ExpectNoError(err)
hostExec := storageutils.NewHostExec(f)
ginkgo.DeferCleanup(hostExec.Cleanup)
// This is a hacky check that greps the container runtime config to determine if the test-handler is the underlying container runtime config.
// For containerd, this is configured in kube-up for GCE clusters here: https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
// For cri-o, see configuration here: https://github.com/cri-o/cri-o/blob/5dc6a035c940f5dde3a727b2e2d8d4b7e371ad55/contrib/test/ci/e2e-base.yml#L2-L13
// If the `runtimes.test-handler` substring is found in the runtime config, it is assumed that the handler is configured.
cmd := fmt.Sprintf(`if [ -e '/etc/containerd/config.toml' ]; then
grep -q 'runtimes.%s' /etc/containerd/config.toml
return
fi
if [ -e '/etc/crio/crio.conf' ]; then
grep -q 'runtimes.%s' /etc/crio/crio.conf
return
fi
`, PreconfiguredRuntimeClassHandler, PreconfiguredRuntimeClassHandler)
_, err = hostExec.IssueCommandWithResult(ctx, cmd, node)
return err
}

View File

@ -30,6 +30,7 @@ import (
runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
"k8s.io/kubernetes/test/e2e/scheduling"
@ -40,7 +41,7 @@ import (
var _ = SIGDescribe("RuntimeClass", func() {
f := framework.NewDefaultFramework("runtimeclass")
f.NamespacePodSecurityEnforceLevel = api.LevelBaseline
f.NamespacePodSecurityEnforceLevel = api.LevelPrivileged
ginkgo.It("should reject a Pod requesting a RuntimeClass with conflicting node selector", func(ctx context.Context) {
labelFooName := "foo-" + string(uuid.NewUUID())
@ -56,7 +57,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
pod := e2enode.NewRuntimeClassPod(rc.GetName())
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
pod.Spec.NodeSelector = map[string]string{
labelFooName: "bar",
}
@ -111,7 +112,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
pod := e2enode.NewRuntimeClassPod(rc.GetName())
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
pod.Spec.NodeSelector = map[string]string{
labelFooName: "bar",
}
@ -128,9 +129,9 @@ var _ = SIGDescribe("RuntimeClass", func() {
})
ginkgo.It("should run a Pod requesting a RuntimeClass with scheduling without taints ", func(ctx context.Context) {
// Requires special setup of test-handler which is only done in GCE kube-up environment
// see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
e2eskipper.SkipUnlessProviderIs("gce")
if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
}
labelFooName := "foo-" + string(uuid.NewUUID())
labelFizzName := "fizz-" + string(uuid.NewUUID())
@ -157,7 +158,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
pod := e2enode.NewRuntimeClassPod(rc.GetName())
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
pod.Spec.NodeSelector = map[string]string{
labelFooName: "bar",
}
@ -176,5 +177,5 @@ var _ = SIGDescribe("RuntimeClass", func() {
// newRuntimeClass returns a test runtime class.
func newRuntimeClass(namespace, name string) *nodev1.RuntimeClass {
uniqueName := fmt.Sprintf("%s-%s", namespace, name)
return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler)
return runtimeclasstest.NewRuntimeClass(uniqueName, e2eruntimeclass.PreconfiguredRuntimeClassHandler)
}

View File

@ -36,6 +36,7 @@ import (
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
@ -238,7 +239,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName)
// Register a runtimeClass with overhead set as 25% of the available beard-seconds
handler = e2enode.PreconfiguredRuntimeClassHandler
handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler
rc := &nodev1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: handler},
@ -270,7 +271,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
}
// remove RuntimeClass
_ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2enode.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{})
_ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2eruntimeclass.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{})
})
ginkgo.It("verify pod overhead is accounted for", func(ctx context.Context) {

View File

@ -76,11 +76,18 @@ func (h *hostExecutor) launchNodeExecPod(ctx context.Context, node string) *v1.P
hostExecPod := e2epod.NewExecPodSpec(ns.Name, "", true)
hostExecPod.GenerateName = fmt.Sprintf("hostexec-%s-", node)
// Use NodeAffinity instead of NodeName so that pods will not
// be immediately Failed by kubelet if it's out of space. Instead
// Pods will be pending in the scheduler until there is space freed
// up.
e2epod.SetNodeAffinity(&hostExecPod.Spec, node)
if framework.TestContext.NodeE2E {
// E2E node tests do not run a scheduler, so set the node name directly
hostExecPod.Spec.NodeName = node
} else {
// Use NodeAffinity instead of NodeName so that pods will not
// be immediately Failed by kubelet if it's out of space. Instead
// Pods will be pending in the scheduler until there is space freed
// up.
e2epod.SetNodeAffinity(&hostExecPod.Spec, node)
}
hostExecPod.Spec.Volumes = []v1.Volume{
{
// Required to enter into host mount namespace via nsenter.

View File

@ -26,7 +26,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
@ -106,7 +106,7 @@ var _ = SIGDescribe("Kubelet PodOverhead handling [LinuxOnly]", func() {
handler string
)
ginkgo.By("Creating a RuntimeClass with Overhead definied", func() {
handler = e2enode.PreconfiguredRuntimeClassHandler
handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler
rc := &nodev1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: handler},
Handler: handler,