mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-19 09:52:49 +00:00
Merge pull request #115219 from bobbypage/gh-108804-skipper
e2e node: Update runtime class handler skip logic
This commit is contained in:
commit
cec91d9c00
@ -35,7 +35,7 @@ import (
|
||||
runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
@ -45,7 +45,7 @@ import (
|
||||
|
||||
var _ = SIGDescribe("RuntimeClass", func() {
|
||||
f := framework.NewDefaultFramework("runtimeclass")
|
||||
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
|
||||
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
|
||||
|
||||
/*
|
||||
Release: v1.20
|
||||
@ -54,7 +54,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
*/
|
||||
framework.ConformanceIt("should reject a Pod requesting a non-existent RuntimeClass [NodeConformance]", func(ctx context.Context) {
|
||||
rcName := f.Namespace.Name + "-nonexistent"
|
||||
expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName))
|
||||
expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
|
||||
})
|
||||
|
||||
// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler not being installed.
|
||||
@ -62,7 +62,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
handler := f.Namespace.Name + "-handler"
|
||||
rcName := createRuntimeClass(ctx, f, "unconfigured-handler", handler, nil)
|
||||
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
|
||||
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
|
||||
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
|
||||
eventSelector := fields.Set{
|
||||
"involvedObject.kind": "Pod",
|
||||
"involvedObject.name": pod.Name,
|
||||
@ -83,13 +83,13 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
// This test requires that the PreconfiguredRuntimeClassHandler has already been set up on nodes.
|
||||
// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler installed and working.
|
||||
ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func(ctx context.Context) {
|
||||
// Requires special setup of test-handler which is only done in GCE kube-up environment
|
||||
// see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
|
||||
e2eskipper.SkipUnlessProviderIs("gce")
|
||||
if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
|
||||
e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
|
||||
}
|
||||
|
||||
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil)
|
||||
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
|
||||
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
|
||||
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
|
||||
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
|
||||
expectPodSuccess(ctx, f, pod)
|
||||
})
|
||||
|
||||
@ -102,9 +102,9 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
is not being tested here.
|
||||
*/
|
||||
framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass without PodOverhead [NodeConformance]", func(ctx context.Context) {
|
||||
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, nil)
|
||||
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
|
||||
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
|
||||
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
|
||||
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
|
||||
// there is only one pod in the namespace
|
||||
label := labels.SelectorFromSet(labels.Set(map[string]string{}))
|
||||
pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
|
||||
@ -127,14 +127,14 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
is not being tested here.
|
||||
*/
|
||||
framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass and initialize its Overhead [NodeConformance]", func(ctx context.Context) {
|
||||
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2enode.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
|
||||
rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
|
||||
PodFixed: v1.ResourceList{
|
||||
v1.ResourceName(v1.ResourceCPU): resource.MustParse("10m"),
|
||||
v1.ResourceName(v1.ResourceMemory): resource.MustParse("1Mi"),
|
||||
},
|
||||
})
|
||||
ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
|
||||
pod := e2epod.NewPodClient(f).Create(ctx, e2enode.NewRuntimeClassPod(rcName))
|
||||
pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
|
||||
// there is only one pod in the namespace
|
||||
label := labels.SelectorFromSet(labels.Set(map[string]string{}))
|
||||
pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
|
||||
@ -174,7 +174,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
}))
|
||||
})
|
||||
|
||||
expectPodRejection(ctx, f, e2enode.NewRuntimeClassPod(rcName))
|
||||
expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
|
||||
})
|
||||
|
||||
/*
|
||||
|
@ -1,51 +0,0 @@
|
||||
/*
|
||||
Copyright 2020 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package node
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
utilpointer "k8s.io/utils/pointer"
|
||||
)
|
||||
|
||||
const (
|
||||
// PreconfiguredRuntimeClassHandler is the name of the runtime handler
|
||||
// that is expected to be preconfigured in the test environment.
|
||||
PreconfiguredRuntimeClassHandler = "test-handler"
|
||||
)
|
||||
|
||||
// NewRuntimeClassPod returns a test pod with the given runtimeClassName
|
||||
func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
RuntimeClassName: &runtimeClassName,
|
||||
Containers: []v1.Container{{
|
||||
Name: "test",
|
||||
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
||||
Command: []string{"true"},
|
||||
}},
|
||||
RestartPolicy: v1.RestartPolicyNever,
|
||||
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
|
||||
},
|
||||
}
|
||||
}
|
83
test/e2e/framework/node/runtimeclass/runtimeclass.go
Normal file
83
test/e2e/framework/node/runtimeclass/runtimeclass.go
Normal file
@ -0,0 +1,83 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package runtimeclass
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
utilpointer "k8s.io/utils/pointer"
|
||||
)
|
||||
|
||||
const (
|
||||
// PreconfiguredRuntimeClassHandler is the name of the runtime handler
|
||||
// that is expected to be preconfigured in the test environment.
|
||||
PreconfiguredRuntimeClassHandler = "test-handler"
|
||||
)
|
||||
|
||||
// NewRuntimeClassPod returns a test pod with the given runtimeClassName
|
||||
func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
RuntimeClassName: &runtimeClassName,
|
||||
Containers: []v1.Container{{
|
||||
Name: "test",
|
||||
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
||||
Command: []string{"true"},
|
||||
}},
|
||||
RestartPolicy: v1.RestartPolicyNever,
|
||||
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Check if test-handler is configured by reading the configuration from container runtime config.
|
||||
// If no error is returned, the container runtime is assumed to support the test-handler, otherwise an error will be returned.
|
||||
func NodeSupportsPreconfiguredRuntimeClassHandler(ctx context.Context, f *framework.Framework) error {
|
||||
node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
|
||||
framework.ExpectNoError(err)
|
||||
hostExec := storageutils.NewHostExec(f)
|
||||
ginkgo.DeferCleanup(hostExec.Cleanup)
|
||||
|
||||
// This is a hacky check that greps the container runtime config to determine if the test-handler is the underlying container runtime config.
|
||||
// For containerd, this is configured in kube-up for GCE clusters here: https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
|
||||
// For cri-o, see configuration here: https://github.com/cri-o/cri-o/blob/5dc6a035c940f5dde3a727b2e2d8d4b7e371ad55/contrib/test/ci/e2e-base.yml#L2-L13
|
||||
// If the `runtimes.test-handler` substring is found in the runtime config, it is assumed that the handler is configured.
|
||||
cmd := fmt.Sprintf(`if [ -e '/etc/containerd/config.toml' ]; then
|
||||
grep -q 'runtimes.%s' /etc/containerd/config.toml
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -e '/etc/crio/crio.conf' ]; then
|
||||
grep -q 'runtimes.%s' /etc/crio/crio.conf
|
||||
return
|
||||
fi
|
||||
`, PreconfiguredRuntimeClassHandler, PreconfiguredRuntimeClassHandler)
|
||||
|
||||
_, err = hostExec.IssueCommandWithResult(ctx, cmd, node)
|
||||
return err
|
||||
}
|
@ -30,6 +30,7 @@ import (
|
||||
runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
"k8s.io/kubernetes/test/e2e/scheduling"
|
||||
@ -40,7 +41,7 @@ import (
|
||||
|
||||
var _ = SIGDescribe("RuntimeClass", func() {
|
||||
f := framework.NewDefaultFramework("runtimeclass")
|
||||
f.NamespacePodSecurityEnforceLevel = api.LevelBaseline
|
||||
f.NamespacePodSecurityEnforceLevel = api.LevelPrivileged
|
||||
|
||||
ginkgo.It("should reject a Pod requesting a RuntimeClass with conflicting node selector", func(ctx context.Context) {
|
||||
labelFooName := "foo-" + string(uuid.NewUUID())
|
||||
@ -56,7 +57,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
|
||||
|
||||
pod := e2enode.NewRuntimeClassPod(rc.GetName())
|
||||
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
|
||||
pod.Spec.NodeSelector = map[string]string{
|
||||
labelFooName: "bar",
|
||||
}
|
||||
@ -111,7 +112,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
|
||||
|
||||
pod := e2enode.NewRuntimeClassPod(rc.GetName())
|
||||
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
|
||||
pod.Spec.NodeSelector = map[string]string{
|
||||
labelFooName: "bar",
|
||||
}
|
||||
@ -128,9 +129,9 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
})
|
||||
|
||||
ginkgo.It("should run a Pod requesting a RuntimeClass with scheduling without taints ", func(ctx context.Context) {
|
||||
// Requires special setup of test-handler which is only done in GCE kube-up environment
|
||||
// see https://github.com/kubernetes/kubernetes/blob/eb729620c522753bc7ae61fc2c7b7ea19d4aad2f/cluster/gce/gci/configure-helper.sh#L3069-L3076
|
||||
e2eskipper.SkipUnlessProviderIs("gce")
|
||||
if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
|
||||
e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
|
||||
}
|
||||
|
||||
labelFooName := "foo-" + string(uuid.NewUUID())
|
||||
labelFizzName := "fizz-" + string(uuid.NewUUID())
|
||||
@ -157,7 +158,7 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
|
||||
|
||||
pod := e2enode.NewRuntimeClassPod(rc.GetName())
|
||||
pod := e2eruntimeclass.NewRuntimeClassPod(rc.GetName())
|
||||
pod.Spec.NodeSelector = map[string]string{
|
||||
labelFooName: "bar",
|
||||
}
|
||||
@ -176,5 +177,5 @@ var _ = SIGDescribe("RuntimeClass", func() {
|
||||
// newRuntimeClass returns a test runtime class.
|
||||
func newRuntimeClass(namespace, name string) *nodev1.RuntimeClass {
|
||||
uniqueName := fmt.Sprintf("%s-%s", namespace, name)
|
||||
return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler)
|
||||
return runtimeclasstest.NewRuntimeClass(uniqueName, e2eruntimeclass.PreconfiguredRuntimeClassHandler)
|
||||
}
|
||||
|
@ -36,6 +36,7 @@ import (
|
||||
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
@ -238,7 +239,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
|
||||
framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName)
|
||||
|
||||
// Register a runtimeClass with overhead set as 25% of the available beard-seconds
|
||||
handler = e2enode.PreconfiguredRuntimeClassHandler
|
||||
handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler
|
||||
|
||||
rc := &nodev1.RuntimeClass{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: handler},
|
||||
@ -270,7 +271,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
|
||||
}
|
||||
|
||||
// remove RuntimeClass
|
||||
_ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2enode.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{})
|
||||
_ = cs.NodeV1beta1().RuntimeClasses().Delete(ctx, e2eruntimeclass.PreconfiguredRuntimeClassHandler, metav1.DeleteOptions{})
|
||||
})
|
||||
|
||||
ginkgo.It("verify pod overhead is accounted for", func(ctx context.Context) {
|
||||
|
@ -76,11 +76,18 @@ func (h *hostExecutor) launchNodeExecPod(ctx context.Context, node string) *v1.P
|
||||
|
||||
hostExecPod := e2epod.NewExecPodSpec(ns.Name, "", true)
|
||||
hostExecPod.GenerateName = fmt.Sprintf("hostexec-%s-", node)
|
||||
// Use NodeAffinity instead of NodeName so that pods will not
|
||||
// be immediately Failed by kubelet if it's out of space. Instead
|
||||
// Pods will be pending in the scheduler until there is space freed
|
||||
// up.
|
||||
e2epod.SetNodeAffinity(&hostExecPod.Spec, node)
|
||||
|
||||
if framework.TestContext.NodeE2E {
|
||||
// E2E node tests do not run a scheduler, so set the node name directly
|
||||
hostExecPod.Spec.NodeName = node
|
||||
} else {
|
||||
// Use NodeAffinity instead of NodeName so that pods will not
|
||||
// be immediately Failed by kubelet if it's out of space. Instead
|
||||
// Pods will be pending in the scheduler until there is space freed
|
||||
// up.
|
||||
e2epod.SetNodeAffinity(&hostExecPod.Spec, node)
|
||||
|
||||
}
|
||||
hostExecPod.Spec.Volumes = []v1.Volume{
|
||||
{
|
||||
// Required to enter into host mount namespace via nsenter.
|
||||
|
@ -26,7 +26,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cm"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
@ -106,7 +106,7 @@ var _ = SIGDescribe("Kubelet PodOverhead handling [LinuxOnly]", func() {
|
||||
handler string
|
||||
)
|
||||
ginkgo.By("Creating a RuntimeClass with Overhead definied", func() {
|
||||
handler = e2enode.PreconfiguredRuntimeClassHandler
|
||||
handler = e2eruntimeclass.PreconfiguredRuntimeClassHandler
|
||||
rc := &nodev1.RuntimeClass{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: handler},
|
||||
Handler: handler,
|
||||
|
Loading…
Reference in New Issue
Block a user