mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-22 09:40:07 +00:00
1847d5b1a2
The "create 100 slices" E2E sometimes flaked with timeouts (e.g. 95 out of 100
slices created). It created too much load for an E2E test.
The same test now uses ktesting as API, which makes it possible to run it as
integration test with the original 100 slices and with more moderate 10 slices
as E2E test.
(cherry picked from commit c47ad64820)
3084 lines
126 KiB
Go
3084 lines
126 KiB
Go
/*
|
|
Copyright 2022 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package dra
|
|
|
|
import (
|
|
"context"
|
|
_ "embed"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/onsi/ginkgo/v2"
|
|
"github.com/onsi/gomega"
|
|
"github.com/onsi/gomega/gstruct"
|
|
"github.com/onsi/gomega/types"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
resourceapi "k8s.io/api/resource/v1"
|
|
resourcealphaapi "k8s.io/api/resource/v1alpha3"
|
|
resourcev1beta1 "k8s.io/api/resource/v1beta1"
|
|
resourcev1beta2 "k8s.io/api/resource/v1beta2"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/fields"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
applyv1 "k8s.io/client-go/applyconfigurations/core/v1"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/klog/v2"
|
|
"k8s.io/kubernetes/pkg/features"
|
|
"k8s.io/kubernetes/pkg/kubelet/events"
|
|
testdriverapp "k8s.io/kubernetes/test/e2e/dra/test-driver/app"
|
|
drautils "k8s.io/kubernetes/test/e2e/dra/utils"
|
|
"k8s.io/kubernetes/test/e2e/feature"
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
e2econformance "k8s.io/kubernetes/test/e2e/framework/conformance"
|
|
e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset"
|
|
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
|
|
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
|
dratest "k8s.io/kubernetes/test/integration/dra"
|
|
"k8s.io/kubernetes/test/utils/ktesting"
|
|
admissionapi "k8s.io/pod-security-admission/api"
|
|
"k8s.io/utils/ptr"
|
|
)
|
|
|
|
const (
|
|
// podStartTimeout is how long to wait for the pod to be started.
|
|
podStartTimeout = 5 * time.Minute
|
|
)
|
|
|
|
// The "DRA" label is used to select tests related to DRA in a Ginkgo label filter.
|
|
//
|
|
// Sub-tests starting with "control plane" when testing only the control plane components, without depending
|
|
// on DRA support in the kubelet.
|
|
//
|
|
// Sub-tests starting with "kubelet" depend on DRA and plugin support in the kubelet.
|
|
var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
|
|
f := framework.NewDefaultFramework("dra")
|
|
|
|
// The driver containers have to run with sufficient privileges to
|
|
// modify /var/lib/kubelet/plugins.
|
|
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
|
|
|
|
f.Context("CRUD Tests", func() {
|
|
/*
|
|
Release: v1.35
|
|
Testname: CRUD operations for deviceclasses
|
|
Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 DeviceClass.
|
|
*/
|
|
framework.ConformanceIt("resource.k8s.io/v1 DeviceClass", func(ctx context.Context) {
|
|
e2econformance.TestResource(ctx, f,
|
|
&e2econformance.ResourceTestcase[*resourceapi.DeviceClass]{
|
|
GVR: resourceapi.SchemeGroupVersion.WithResource("deviceclasses"),
|
|
Namespaced: ptr.To(false),
|
|
InitialSpec: &resourceapi.DeviceClass{
|
|
Spec: resourceapi.DeviceClassSpec{
|
|
Selectors: []resourceapi.DeviceSelector{{
|
|
CEL: &resourceapi.CELDeviceSelector{
|
|
Expression: "false", // Matches no devices
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
UpdateSpec: func(obj *resourceapi.DeviceClass) *resourceapi.DeviceClass {
|
|
obj.Spec.Selectors[0].CEL.Expression = "1 == 0" // Still matches no devices.
|
|
return obj
|
|
},
|
|
StrategicMergePatchSpec: `{"spec": {"selectors": [{"cel": {"expression": "1 == 0"}}]}}`,
|
|
},
|
|
)
|
|
})
|
|
|
|
/*
|
|
Release: v1.35
|
|
Testname: CRUD operations for resourceclaims
|
|
Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceClaim.
|
|
*/
|
|
framework.ConformanceIt("resource.k8s.io/v1 ResourceClaim", func(ctx context.Context) {
|
|
e2econformance.TestResource(ctx, f,
|
|
&e2econformance.ResourceTestcase[*resourceapi.ResourceClaim]{
|
|
GVR: resourceapi.SchemeGroupVersion.WithResource("resourceclaims"),
|
|
Namespaced: ptr.To(true),
|
|
InitialSpec: &resourceapi.ResourceClaim{
|
|
Spec: resourceapi.ResourceClaimSpec{
|
|
Devices: resourceapi.DeviceClaim{
|
|
Requests: []resourceapi.DeviceRequest{{
|
|
Name: "req-0",
|
|
Exactly: &resourceapi.ExactDeviceRequest{
|
|
DeviceClassName: "dra.example.com",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
UpdateSpec: func(obj *resourceapi.ResourceClaim) *resourceapi.ResourceClaim {
|
|
// The spec is immutable, so let's add a label instead.
|
|
if obj.Labels == nil {
|
|
obj.Labels = make(map[string]string)
|
|
}
|
|
obj.Labels["test.dra.example.com"] = "test"
|
|
return obj
|
|
},
|
|
UpdateStatus: func(obj *resourceapi.ResourceClaim) *resourceapi.ResourceClaim {
|
|
// Nothing allocated" is a valid allocation result.
|
|
obj.Status.Allocation = &resourceapi.AllocationResult{}
|
|
return obj
|
|
},
|
|
|
|
// This test case uses all available patch types to demonstrate what this looks like
|
|
// and that TestResource supports them. Strategic merge patch, apply patch, and JSON
|
|
// merge patch are identical for these simple modifications.
|
|
//
|
|
// Testing with only one patch type would be sufficient for conformance.
|
|
ApplyPatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`,
|
|
StrategicMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`,
|
|
JSONMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`,
|
|
JSONPatchSpec: `[{"op": "add", "path": "/metadata/labels/test.dra.example.com", "value": "test"}]`,
|
|
ApplyPatchStatus: `{"status": {"allocation": {}}}`,
|
|
StrategicMergePatchStatus: `{"status": {"allocation": {}}}`,
|
|
JSONMergePatchStatus: `{"status": {"allocation": {}}}`,
|
|
JSONPatchStatus: `[{"op": "add", "path": "/status/allocation", "value": {}}]`,
|
|
})
|
|
})
|
|
|
|
/*
|
|
Release: v1.35
|
|
Testname: CRUD operations for resourceclaimtemplates
|
|
Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceClaimTemplate.
|
|
*/
|
|
framework.ConformanceIt("resource.k8s.io/v1 ResourceClaimTemplate", func(ctx context.Context) {
|
|
e2econformance.TestResource(ctx, f,
|
|
&e2econformance.ResourceTestcase[*resourceapi.ResourceClaimTemplate]{
|
|
GVR: resourceapi.SchemeGroupVersion.WithResource("resourceclaimtemplates"),
|
|
Namespaced: ptr.To(true),
|
|
InitialSpec: &resourceapi.ResourceClaimTemplate{
|
|
Spec: resourceapi.ResourceClaimTemplateSpec{
|
|
Spec: resourceapi.ResourceClaimSpec{
|
|
Devices: resourceapi.DeviceClaim{
|
|
Requests: []resourceapi.DeviceRequest{{
|
|
Name: "req-0",
|
|
Exactly: &resourceapi.ExactDeviceRequest{
|
|
DeviceClassName: "dra.example.com",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
UpdateSpec: func(obj *resourceapi.ResourceClaimTemplate) *resourceapi.ResourceClaimTemplate {
|
|
// The spec is immutable, so let's add a label instead.
|
|
if obj.Labels == nil {
|
|
obj.Labels = make(map[string]string)
|
|
}
|
|
obj.Labels["test.dra.example.com"] = "test"
|
|
return obj
|
|
},
|
|
StrategicMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`,
|
|
},
|
|
)
|
|
})
|
|
|
|
/*
|
|
Release: v1.35
|
|
Testname: CRUD operations for resoureslices
|
|
Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceSlice.
|
|
*/
|
|
framework.ConformanceIt("resource.k8s.io/v1 ResourceSlice", func(ctx context.Context) {
|
|
e2econformance.TestResource(ctx, f,
|
|
&e2econformance.ResourceTestcase[*resourceapi.ResourceSlice]{
|
|
GVR: resourceapi.SchemeGroupVersion.WithResource("resourceslices"),
|
|
Namespaced: ptr.To(false),
|
|
InitialSpec: &resourceapi.ResourceSlice{
|
|
Spec: resourceapi.ResourceSliceSpec{
|
|
Driver: "dra.example.com",
|
|
Pool: resourceapi.ResourcePool{
|
|
Name: "cluster",
|
|
Generation: 1,
|
|
ResourceSliceCount: 1,
|
|
},
|
|
NodeName: ptr.To("no-such-node"),
|
|
// The pool is empty -> no devices.
|
|
},
|
|
},
|
|
UpdateSpec: func(obj *resourceapi.ResourceSlice) *resourceapi.ResourceSlice {
|
|
obj.Spec.Devices = []resourceapi.Device{{Name: "device-0"}}
|
|
return obj
|
|
},
|
|
StrategicMergePatchSpec: `{"spec": {"devices": [{"name": "device-0"}]}}`,
|
|
},
|
|
)
|
|
})
|
|
})
|
|
|
|
f.Context("kubelet", feature.DynamicResourceAllocation, func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false))
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
ginkgo.It("registers plugin", func() {
|
|
ginkgo.By("the driver is running")
|
|
})
|
|
|
|
ginkgo.It("must retry NodePrepareResources", func(ctx context.Context) {
|
|
// We have exactly one host. The API version depends on the kubelet
|
|
// we test with (version skew!), so we need to be a bit flexible.
|
|
mV1Beta1 := drautils.MethodInstance{NodeName: driver.Nodenames()[0], FullMethod: "/k8s.io.kubelet.pkg.apis.dra.v1beta1.DRAPlugin/NodePrepareResources"}
|
|
mV1 := drautils.MethodInstance{NodeName: driver.Nodenames()[0], FullMethod: "/k8s.io.kubelet.pkg.apis.dra.v1.DRAPlugin/NodePrepareResources"}
|
|
|
|
driver.Fail(mV1Beta1, true)
|
|
driver.Fail(mV1, true)
|
|
|
|
ginkgo.By("waiting for container startup to fail")
|
|
pod, template := b.PodInline()
|
|
|
|
b.Create(f.TContext(ctx), pod, template)
|
|
|
|
ginkgo.By("wait for NodePrepareResources call")
|
|
gomega.Eventually(ctx, func(ctx context.Context) error {
|
|
if driver.CallCount(mV1Beta1)+driver.CallCount(mV1) == 0 {
|
|
return errors.New("NodePrepareResources not called yet")
|
|
}
|
|
return nil
|
|
}).WithTimeout(podStartTimeout).Should(gomega.Succeed())
|
|
|
|
ginkgo.By("allowing container startup to succeed")
|
|
callCount := driver.CallCount(mV1Beta1) + driver.CallCount(mV1)
|
|
driver.Fail(mV1, false)
|
|
driver.Fail(mV1Beta1, false)
|
|
err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace)
|
|
framework.ExpectNoError(err, "start pod with inline resource claim")
|
|
if driver.CallCount(mV1Beta1)+driver.CallCount(mV1) == callCount {
|
|
framework.Fail("NodePrepareResources should have been called again")
|
|
}
|
|
})
|
|
|
|
ginkgo.It("must not run a pod if a claim is not ready", func(ctx context.Context) {
|
|
claim := b.ExternalClaim()
|
|
b.Create(f.TContext(ctx), claim)
|
|
pod := b.PodExternal()
|
|
|
|
// This bypasses scheduling and therefore the pod gets
|
|
// to run on the node although the claim is not ready.
|
|
// Because the parameters are missing, the claim
|
|
// also cannot be allocated later.
|
|
pod.Spec.NodeName = nodes.NodeNames[0]
|
|
b.Create(f.TContext(ctx), pod)
|
|
|
|
gomega.Consistently(ctx, func(ctx context.Context) error {
|
|
testPod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("expected the test pod %s to exist: %w", pod.Name, err)
|
|
}
|
|
if testPod.Status.Phase != v1.PodPending {
|
|
return fmt.Errorf("pod %s: unexpected status %s, expected status: %s", pod.Name, testPod.Status.Phase, v1.PodPending)
|
|
}
|
|
return nil
|
|
}, 20*time.Second, 200*time.Millisecond).Should(gomega.BeNil())
|
|
})
|
|
|
|
ginkgo.It("must unprepare resources for force-deleted pod", func(ctx context.Context) {
|
|
claim := b.ExternalClaim()
|
|
pod := b.PodExternal()
|
|
zero := int64(0)
|
|
pod.Spec.TerminationGracePeriodSeconds = &zero
|
|
|
|
b.Create(f.TContext(ctx), claim, pod)
|
|
|
|
b.TestPod(f.TContext(ctx), pod)
|
|
|
|
ginkgo.By(fmt.Sprintf("force delete test pod %s", pod.Name))
|
|
err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &zero})
|
|
if !apierrors.IsNotFound(err) {
|
|
framework.ExpectNoError(err, "force delete test pod")
|
|
}
|
|
|
|
for host, plugin := range driver.Nodes {
|
|
ginkgo.By(fmt.Sprintf("waiting for resources on %s to be unprepared", host))
|
|
gomega.Eventually(plugin.GetPreparedResources).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host)
|
|
}
|
|
})
|
|
|
|
ginkgo.It("must call NodePrepareResources even if not used by any container", func(ctx context.Context) {
|
|
pod, template := b.PodInline()
|
|
for i := range pod.Spec.Containers {
|
|
pod.Spec.Containers[i].Resources.Claims = nil
|
|
}
|
|
b.Create(f.TContext(ctx), pod, template)
|
|
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod), "start pod")
|
|
for host, plugin := range driver.Nodes {
|
|
gomega.Expect(plugin.GetPreparedResources()).ShouldNot(gomega.BeEmpty(), "claims should be prepared on host %s while pod is running", host)
|
|
}
|
|
})
|
|
|
|
ginkgo.It("must map configs and devices to the right containers", func(ctx context.Context) {
|
|
// Several claims, each with three requests and three configs.
|
|
// One config applies to all requests, the other two only to one request each.
|
|
claimForAllContainers := b.ExternalClaim()
|
|
claimForAllContainers.Name = "all"
|
|
claimForAllContainers.Spec.Devices.Requests = append(claimForAllContainers.Spec.Devices.Requests,
|
|
*claimForAllContainers.Spec.Devices.Requests[0].DeepCopy(),
|
|
*claimForAllContainers.Spec.Devices.Requests[0].DeepCopy(),
|
|
)
|
|
claimForAllContainers.Spec.Devices.Requests[0].Name = "req0"
|
|
claimForAllContainers.Spec.Devices.Requests[1].Name = "req1"
|
|
claimForAllContainers.Spec.Devices.Requests[2].Name = "req2"
|
|
claimForAllContainers.Spec.Devices.Config = append(claimForAllContainers.Spec.Devices.Config,
|
|
*claimForAllContainers.Spec.Devices.Config[0].DeepCopy(),
|
|
*claimForAllContainers.Spec.Devices.Config[0].DeepCopy(),
|
|
)
|
|
claimForAllContainers.Spec.Devices.Config[0].Requests = nil
|
|
claimForAllContainers.Spec.Devices.Config[1].Requests = []string{"req1"}
|
|
claimForAllContainers.Spec.Devices.Config[2].Requests = []string{"req2"}
|
|
claimForAllContainers.Spec.Devices.Config[0].Opaque.Parameters.Raw = []byte(`{"all_config0":"true"}`)
|
|
claimForAllContainers.Spec.Devices.Config[1].Opaque.Parameters.Raw = []byte(`{"all_config1":"true"}`)
|
|
claimForAllContainers.Spec.Devices.Config[2].Opaque.Parameters.Raw = []byte(`{"all_config2":"true"}`)
|
|
|
|
claimForContainer0 := claimForAllContainers.DeepCopy()
|
|
claimForContainer0.Name = "container0"
|
|
claimForContainer0.Spec.Devices.Config[0].Opaque.Parameters.Raw = []byte(`{"container0_config0":"true"}`)
|
|
claimForContainer0.Spec.Devices.Config[1].Opaque.Parameters.Raw = []byte(`{"container0_config1":"true"}`)
|
|
claimForContainer0.Spec.Devices.Config[2].Opaque.Parameters.Raw = []byte(`{"container0_config2":"true"}`)
|
|
claimForContainer1 := claimForAllContainers.DeepCopy()
|
|
claimForContainer1.Name = "container1"
|
|
claimForContainer1.Spec.Devices.Config[0].Opaque.Parameters.Raw = []byte(`{"container1_config0":"true"}`)
|
|
claimForContainer1.Spec.Devices.Config[1].Opaque.Parameters.Raw = []byte(`{"container1_config1":"true"}`)
|
|
claimForContainer1.Spec.Devices.Config[2].Opaque.Parameters.Raw = []byte(`{"container1_config2":"true"}`)
|
|
|
|
pod := b.PodExternal()
|
|
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
|
{
|
|
Name: "all",
|
|
ResourceClaimName: &claimForAllContainers.Name,
|
|
},
|
|
{
|
|
Name: "container0",
|
|
ResourceClaimName: &claimForContainer0.Name,
|
|
},
|
|
{
|
|
Name: "container1",
|
|
ResourceClaimName: &claimForContainer1.Name,
|
|
},
|
|
}
|
|
|
|
// Add a second container.
|
|
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
|
|
pod.Spec.Containers[0].Name = "container0"
|
|
pod.Spec.Containers[1].Name = "container1"
|
|
|
|
// All claims use unique env variables which can be used to verify that they
|
|
// have been mapped into the right containers. In addition, the test driver
|
|
// also sets "claim_<claim name>_<request name>=true" with non-alphanumeric
|
|
// replaced by underscore.
|
|
|
|
// Both requests (claim_*_req*) and all user configs (user_*_config*).
|
|
allContainersEnv := []string{
|
|
"user_all_config0", "true",
|
|
"user_all_config1", "true",
|
|
"user_all_config2", "true",
|
|
"claim_all_req0", "true",
|
|
"claim_all_req1", "true",
|
|
"claim_all_req2", "true",
|
|
}
|
|
|
|
// Everything from the "all" claim and everything from the "container0" claim.
|
|
pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: "all"}, {Name: "container0"}}
|
|
container0Env := []string{
|
|
"user_container0_config0", "true",
|
|
"user_container0_config1", "true",
|
|
"user_container0_config2", "true",
|
|
"claim_container0_req0", "true",
|
|
"claim_container0_req1", "true",
|
|
"claim_container0_req2", "true",
|
|
}
|
|
container0Env = append(container0Env, allContainersEnv...)
|
|
|
|
// Everything from the "all" claim, but only the second request from the "container1" claim.
|
|
// The first two configs apply.
|
|
pod.Spec.Containers[1].Resources.Claims = []v1.ResourceClaim{{Name: "all"}, {Name: "container1", Request: "req1"}}
|
|
container1Env := []string{
|
|
"user_container1_config0", "true",
|
|
"user_container1_config1", "true",
|
|
// Does not apply: user_container1_config2
|
|
"claim_container1_req1", "true",
|
|
}
|
|
container1Env = append(container1Env, allContainersEnv...)
|
|
|
|
b.Create(f.TContext(ctx), claimForAllContainers, claimForContainer0, claimForContainer1, pod)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
|
|
drautils.TestContainerEnv(f.TContext(ctx), pod, pod.Spec.Containers[0].Name, true, container0Env...)
|
|
drautils.TestContainerEnv(f.TContext(ctx), pod, pod.Spec.Containers[1].Name, true, container1Env...)
|
|
})
|
|
|
|
// https://github.com/kubernetes/kubernetes/issues/131513 was fixed in master for 1.34 and not backported,
|
|
// so this test only passes for kubelet >= 1.34.
|
|
f.It("blocks new pod after force-delete", f.WithLabel("KubeletMinVersion:1.34"), func(ctx context.Context) {
|
|
// The problem with a force-deleted pod is that kubelet
|
|
// is not necessarily done yet with tearing down the
|
|
// pod at the time when the pod and its claim are
|
|
// already removed. The user can replace the claim and
|
|
// pod with new instances under the same name. The
|
|
// kubelet then needs to detect that the new claim is
|
|
// not the same as the one that kubelet currently works
|
|
// on and that the new pod cannot start until the old
|
|
// one is torn down.
|
|
//
|
|
// This test delays termination of the first pod to ensure
|
|
// that the race goes bad (old pod pending shutdown when
|
|
// new one arrives) and always schedules to the same node.
|
|
claim := b.ExternalClaim()
|
|
pod := b.PodExternal()
|
|
node := nodes.NodeNames[0]
|
|
pod.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": node}
|
|
oldClaim := b.Create(f.TContext(ctx), claim, pod)[0].(*resourceapi.ResourceClaim)
|
|
b.TestPod(f.TContext(ctx), pod)
|
|
|
|
ginkgo.By("Force-delete claim and pod")
|
|
forceDelete := metav1.DeleteOptions{GracePeriodSeconds: ptr.To(int64(0))}
|
|
framework.ExpectNoError(f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, forceDelete))
|
|
|
|
// Fail NodeUnprepareResources to simulate long grace period
|
|
unprepareResources := drautils.MethodInstance{NodeName: node, FullMethod: "/k8s.io.kubelet.pkg.apis.dra.v1.DRAPlugin/NodeUnprepareResources"}
|
|
driver.Fail(unprepareResources, true)
|
|
|
|
// The pod should get deleted immediately.
|
|
_, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
if !apierrors.IsNotFound(err) {
|
|
framework.Failf("Expected 'not found' error, got: %v", err)
|
|
}
|
|
|
|
// The claim may take a bit longer because of the allocation and finalizer.
|
|
framework.ExpectNoError(f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Delete(ctx, claim.Name, forceDelete))
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
|
claim, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
if apierrors.IsNotFound(err) {
|
|
return nil, nil
|
|
}
|
|
return claim, err
|
|
}).Should(gomega.BeNil())
|
|
gomega.Expect(driver.Nodes[node].GetPreparedResources()).Should(gomega.Equal([]testdriverapp.ClaimID{{Name: oldClaim.Name, UID: oldClaim.UID}}), "Old claim should still be prepared.")
|
|
|
|
ginkgo.By("Re-creating the same claim and pod")
|
|
newClaim := b.Create(f.TContext(ctx), claim, pod)[0].(*resourceapi.ResourceClaim)
|
|
|
|
// Keep blocking NodeUnprepareResources for the old pod
|
|
// until the new pod calls NodePrepareResources and fails.
|
|
// This ensures that the race is triggered.
|
|
expectedEvent := fields.Set{
|
|
"involvedObject.kind": "Pod",
|
|
"involvedObject.name": pod.Name,
|
|
"involvedObject.namespace": pod.Namespace,
|
|
"reason": events.FailedPrepareDynamicResources,
|
|
}.AsSelector().String()
|
|
|
|
// 10 min timeout (PodStartTimeout * 2) should be enough
|
|
// for Kubelet to emit multiple events, so the test should
|
|
// be able to catch at least one of them.
|
|
framework.ExpectNoError(e2eevents.WaitTimeoutForEvent(
|
|
ctx,
|
|
f.ClientSet,
|
|
pod.Namespace,
|
|
expectedEvent,
|
|
fmt.Sprintf("old ResourceClaim with same name %s and different UID %s still exists", oldClaim.Name, oldClaim.UID),
|
|
framework.PodStartTimeout*2))
|
|
|
|
driver.Fail(unprepareResources, false)
|
|
|
|
b.TestPod(f.TContext(ctx), pod)
|
|
|
|
// The pod must not have started before NodeUnprepareResources was called for the old one,
|
|
// i.e. what is prepared now must be the new claim.
|
|
gomega.Expect(driver.Nodes[node].GetPreparedResources()).Should(gomega.Equal([]testdriverapp.ClaimID{{Name: newClaim.Name, UID: newClaim.UID}}), "Only new claim should be prepared now because new pod is running.")
|
|
})
|
|
|
|
f.It("DaemonSet with admin access", f.WithFeatureGate(features.DRAAdminAccess), func(ctx context.Context) {
|
|
// Ensure namespace has the dra admin label.
|
|
_, err := f.ClientSet.CoreV1().Namespaces().Apply(ctx,
|
|
applyv1.Namespace(f.Namespace.Name).WithLabels(map[string]string{"resource.kubernetes.io/admin-access": "true"}),
|
|
metav1.ApplyOptions{FieldManager: f.UniqueName})
|
|
framework.ExpectNoError(err)
|
|
|
|
pod, template := b.PodInline()
|
|
template.Spec.Spec.Devices.Requests[0].Exactly.AdminAccess = ptr.To(true)
|
|
// Limit the daemon set to the one node where we have the driver.
|
|
nodeName := nodes.NodeNames[0]
|
|
pod.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": nodeName}
|
|
pod.Spec.RestartPolicy = v1.RestartPolicyAlways
|
|
daemonSet := &appsv1.DaemonSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "monitoring-ds",
|
|
Namespace: f.Namespace.Name,
|
|
},
|
|
Spec: appsv1.DaemonSetSpec{
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"app": "monitoring"},
|
|
},
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"app": "monitoring"},
|
|
},
|
|
Spec: pod.Spec,
|
|
},
|
|
},
|
|
}
|
|
|
|
created := b.Create(f.TContext(ctx), template, daemonSet)
|
|
if !ptr.Deref(created[0].(*resourceapi.ResourceClaimTemplate).Spec.Spec.Devices.Requests[0].Exactly.AdminAccess, false) {
|
|
framework.Fail("AdminAccess field was cleared. This test depends on the DRAAdminAccess feature.")
|
|
}
|
|
ds := created[1].(*appsv1.DaemonSet)
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) (bool, error) {
|
|
return e2edaemonset.CheckDaemonPodOnNodes(f, ds, []string{nodeName})(ctx)
|
|
}).WithTimeout(f.Timeouts.PodStart).Should(gomega.BeTrueBecause("DaemonSet pod should be running on node %s but isn't", nodeName))
|
|
framework.ExpectNoError(e2edaemonset.CheckDaemonStatus(ctx, f, daemonSet.Name))
|
|
})
|
|
})
|
|
|
|
// Same "kubelet" context as above, but now with per-node resources.
|
|
f.Context("kubelet", feature.DynamicResourceAllocation, func() {
|
|
nodes := drautils.NewNodes(f, 1, 4)
|
|
driver := drautils.NewDriver(f, nodes, drautils.DriverResources(1))
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
f.It("must manage ResourceSlices", func(ctx context.Context) {
|
|
driverName := driver.Name
|
|
|
|
// Now check for exactly the right set of objects for all nodes.
|
|
ginkgo.By("check if ResourceSlice object(s) exist on the API server")
|
|
resourceClient := f.ClientSet.ResourceV1().ResourceSlices()
|
|
var expectedObjects []any
|
|
for _, nodeName := range nodes.NodeNames {
|
|
node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
|
|
framework.ExpectNoError(err, "get node")
|
|
expectedObjects = append(expectedObjects,
|
|
gstruct.MatchAllFields(gstruct.Fields{
|
|
"TypeMeta": gstruct.Ignore(),
|
|
"ObjectMeta": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"OwnerReferences": gomega.ContainElements(
|
|
gstruct.MatchAllFields(gstruct.Fields{
|
|
"APIVersion": gomega.Equal("v1"),
|
|
"Kind": gomega.Equal("Node"),
|
|
"Name": gomega.Equal(nodeName),
|
|
"UID": gomega.Equal(node.UID),
|
|
"Controller": gomega.Equal(ptr.To(true)),
|
|
"BlockOwnerDeletion": gomega.BeNil(),
|
|
}),
|
|
),
|
|
}),
|
|
// Ignoring some fields, like SharedCounters, because we don't run this test
|
|
// for PRs (it's slow) and don't want CI breaks when fields get added or renamed.
|
|
"Spec": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Driver": gomega.Equal(driver.Name),
|
|
"NodeName": gomega.Equal(ptr.To(nodeName)),
|
|
"NodeSelector": gomega.BeNil(),
|
|
"AllNodes": gomega.BeNil(),
|
|
"Pool": gstruct.MatchAllFields(gstruct.Fields{
|
|
"Name": gomega.Equal(nodeName),
|
|
"Generation": gstruct.Ignore(),
|
|
"ResourceSliceCount": gomega.Equal(int64(1)),
|
|
}),
|
|
"Devices": gomega.Equal([]resourceapi.Device{{Name: "device-00"}}),
|
|
}),
|
|
}),
|
|
)
|
|
}
|
|
matchSlices := gomega.ContainElements(expectedObjects...)
|
|
getSlices := func(ctx context.Context) ([]resourceapi.ResourceSlice, error) {
|
|
slices, err := resourceClient.List(ctx, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return slices.Items, nil
|
|
}
|
|
gomega.Eventually(ctx, getSlices).WithTimeout(20 * time.Second).Should(matchSlices)
|
|
gomega.Consistently(ctx, getSlices).WithTimeout(20 * time.Second).Should(matchSlices)
|
|
|
|
// Removal of node resource slice is tested by the general driver removal code.
|
|
})
|
|
|
|
ginkgo.It("supports init containers with external claims", func(ctx context.Context) {
|
|
pod := b.PodExternal()
|
|
claim := b.ExternalClaim()
|
|
pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]}
|
|
pod.Spec.InitContainers[0].Name += "-init"
|
|
// This must succeed for the pod to start.
|
|
pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env | grep user_a=b"}
|
|
b.Create(f.TContext(ctx), pod, claim)
|
|
|
|
b.TestPod(f.TContext(ctx), pod)
|
|
})
|
|
|
|
ginkgo.It("removes reservation from claim when pod is done", func(ctx context.Context) {
|
|
pod := b.PodExternal()
|
|
claim := b.ExternalClaim()
|
|
pod.Spec.Containers[0].Command = []string{"true"}
|
|
b.Create(f.TContext(ctx), claim, pod)
|
|
|
|
ginkgo.By("waiting for pod to finish")
|
|
framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish")
|
|
ginkgo.By("waiting for claim to be unreserved")
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
|
return f.ClientSet.ResourceV1().ResourceClaims(pod.Namespace).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.ReservedFor", gomega.BeEmpty()), "reservation should have been removed")
|
|
})
|
|
|
|
ginkgo.It("deletes generated claims when pod is done", func(ctx context.Context) {
|
|
pod, template := b.PodInline()
|
|
pod.Spec.Containers[0].Command = []string{"true"}
|
|
b.Create(f.TContext(ctx), template, pod)
|
|
|
|
ginkgo.By("waiting for pod to finish")
|
|
framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish")
|
|
ginkgo.By("waiting for claim to be deleted")
|
|
gomega.Eventually(ctx, func(ctx context.Context) ([]resourceapi.ResourceClaim, error) {
|
|
claims, err := f.ClientSet.ResourceV1().ResourceClaims(pod.Namespace).List(ctx, metav1.ListOptions{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return claims.Items, nil
|
|
}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.BeEmpty(), "claim should have been deleted")
|
|
})
|
|
|
|
ginkgo.It("does not delete generated claims when pod is restarting", func(ctx context.Context) {
|
|
pod, template := b.PodInline()
|
|
pod.Spec.Containers[0].Command = []string{"sh", "-c", "sleep 1; exit 1"}
|
|
pod.Spec.RestartPolicy = v1.RestartPolicyAlways
|
|
b.Create(f.TContext(ctx), template, pod)
|
|
|
|
ginkgo.By("waiting for pod to restart twice")
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*v1.Pod, error) {
|
|
return f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
}).WithTimeout(f.Timeouts.PodStartSlow).Should(gomega.HaveField("Status.ContainerStatuses", gomega.ContainElements(gomega.HaveField("RestartCount", gomega.BeNumerically(">=", 2)))))
|
|
})
|
|
})
|
|
|
|
// kubelet tests with individual configurations.
|
|
f.Context("kubelet", feature.DynamicResourceAllocation, func() {
|
|
ginkgo.It("runs pod after driver starts", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
nodes := drautils.NewNodesNow(tCtx, 1, 4)
|
|
driver := drautils.NewDriverInstance(tCtx)
|
|
b := drautils.NewBuilderNow(tCtx, driver)
|
|
|
|
claim := b.ExternalClaim()
|
|
pod := b.PodExternal()
|
|
b.Create(tCtx, claim, pod)
|
|
|
|
// Cannot run pod, no devices.
|
|
framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
|
|
|
|
// Set up driver, which makes devices available.
|
|
driver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1))
|
|
|
|
// Now it should run.
|
|
b.TestPod(tCtx, pod)
|
|
|
|
// We need to clean up explicitly because the normal
|
|
// cleanup doesn't work (driver shuts down first).
|
|
framework.ExpectNoError(f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}))
|
|
framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodDelete))
|
|
})
|
|
|
|
// Seamless upgrade support was added in Kubernetes 1.33.
|
|
f.It("rolling update", f.WithLabel("KubeletMinVersion:1.33"), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
nodes := drautils.NewNodesNow(tCtx, 1, 1)
|
|
|
|
oldDriver := drautils.NewDriverInstance(tCtx)
|
|
oldDriver.InstanceSuffix = "-old"
|
|
oldDriver.RollingUpdate = true
|
|
oldDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1))
|
|
|
|
// We expect one ResourceSlice per node from the driver.
|
|
getSlices := oldDriver.NewGetSlices()
|
|
ktesting.Eventually(tCtx, getSlices).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames))))
|
|
initialSlices := getSlices(tCtx)
|
|
|
|
// Same driver name, different socket paths because of rolling update.
|
|
newDriver := drautils.NewDriverInstance(tCtx)
|
|
newDriver.InstanceSuffix = "-new"
|
|
newDriver.RollingUpdate = true
|
|
newDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1))
|
|
|
|
// Stop old driver instance.
|
|
oldDriver.TearDown(tCtx)
|
|
|
|
// Build behaves the same for both driver instances.
|
|
b := drautils.NewBuilderNow(tCtx, oldDriver)
|
|
claim := b.ExternalClaim()
|
|
pod := b.PodExternal()
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod)
|
|
|
|
// The exact same slices should still exist.
|
|
finalSlices := getSlices(tCtx)
|
|
gomega.Expect(finalSlices.Items).Should(gomega.Equal(initialSlices.Items))
|
|
|
|
// We need to clean up explicitly because the normal
|
|
// cleanup doesn't work (driver shuts down first).
|
|
framework.ExpectNoError(f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}))
|
|
framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodDelete))
|
|
})
|
|
|
|
// Seamless upgrade support was added in Kubernetes 1.33.
|
|
f.It("failed update", f.WithLabel("KubeletMinVersion:1.33"), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
nodes := drautils.NewNodesNow(tCtx, 1, 1)
|
|
|
|
oldDriver := drautils.NewDriverInstance(tCtx)
|
|
oldDriver.InstanceSuffix = "-old"
|
|
oldDriver.RollingUpdate = true
|
|
oldDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1))
|
|
|
|
// We expect one ResourceSlice per node from the driver.
|
|
getSlices := oldDriver.NewGetSlices()
|
|
ktesting.Eventually(tCtx, getSlices).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames))))
|
|
initialSlices := getSlices(tCtx)
|
|
|
|
// Same driver name, different socket paths because of rolling update.
|
|
newDriver := drautils.NewDriverInstance(tCtx)
|
|
newDriver.InstanceSuffix = "-new"
|
|
newDriver.RollingUpdate = true
|
|
newDriver.ExpectResourceSliceRemoval = false
|
|
newDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1))
|
|
|
|
// Stop new driver instance, simulating the failure of the new instance.
|
|
// The kubelet should still have the old instance.
|
|
newDriver.TearDown(tCtx)
|
|
|
|
// Build behaves the same for both driver instances.
|
|
b := drautils.NewBuilderNow(tCtx, oldDriver)
|
|
claim := b.ExternalClaim()
|
|
pod := b.PodExternal()
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod)
|
|
|
|
// The exact same slices should still exist.
|
|
finalSlices := getSlices(tCtx)
|
|
gomega.Expect(finalSlices.Items).Should(gomega.Equal(initialSlices.Items))
|
|
|
|
// We need to clean up explicitly because the normal
|
|
// cleanup doesn't work (driver shuts down first).
|
|
framework.ExpectNoError(f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}))
|
|
framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodDelete))
|
|
})
|
|
|
|
// Seamless upgrade support was added in Kubernetes 1.33.
|
|
f.It("sequential update with pods replacing each other", f.WithLabel("KubeletMinVersion:1.33"), framework.WithSlow(), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
nodes := drautils.NewNodesNow(tCtx, 1, 1)
|
|
|
|
// Same driver name, same socket path.
|
|
oldDriver := drautils.NewDriverInstance(tCtx)
|
|
oldDriver.InstanceSuffix = "-old"
|
|
oldDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1))
|
|
|
|
// Collect set of resource slices for that driver.
|
|
listSlices := oldDriver.NewGetSlices()
|
|
ktesting.Eventually(tCtx, listSlices).Should(gomega.HaveField("Items", gomega.Not(gomega.BeEmpty())), "driver should have published ResourceSlices, got none")
|
|
oldSlices := listSlices(tCtx)
|
|
if len(oldSlices.Items) == 0 {
|
|
framework.Fail("driver should have published ResourceSlices, got none")
|
|
}
|
|
|
|
// "Update" the driver by taking it down and bringing up a new one.
|
|
// Pods never run in parallel, similar to how a DaemonSet would update
|
|
// its pods when maxSurge is zero.
|
|
ginkgo.By("reinstall driver")
|
|
start := time.Now()
|
|
oldDriver.TearDown(tCtx)
|
|
newDriver := drautils.NewDriverInstance(tCtx)
|
|
newDriver.InstanceSuffix = "-new"
|
|
newDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1))
|
|
updateDuration := time.Since(start)
|
|
|
|
// Build behaves the same for both driver instances.
|
|
b := drautils.NewBuilderNow(tCtx, oldDriver)
|
|
claim := b.ExternalClaim()
|
|
pod := b.PodExternal()
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod)
|
|
|
|
// The slices should have survived the update, but only if it happened
|
|
// quickly enough. If it took too long (= wipingDelay of 30 seconds in pkg/kubelet/cm/dra/manager.go,
|
|
// https://github.com/kubernetes/kubernetes/blob/03763fd1abdf0f5d3dfceb3a6b138bb643e37411/pkg/kubelet/cm/dra/manager.go#L113),
|
|
// the kubelet considered the driver gone and removed them.
|
|
if updateDuration <= 25*time.Second {
|
|
framework.Logf("Checking resource slices after downtime of %s.", updateDuration)
|
|
newSlices := listSlices(tCtx)
|
|
gomega.Expect(newSlices.Items).To(gomega.ConsistOf(oldSlices.Items), "Old slice should have survived a downtime of %s.", updateDuration)
|
|
} else {
|
|
framework.Logf("Not checking resource slices, downtime was too long with %s.", updateDuration)
|
|
}
|
|
|
|
// We need to clean up explicitly because the normal
|
|
// cleanup doesn't work (driver shuts down first).
|
|
framework.ExpectNoError(f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}))
|
|
framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodDelete))
|
|
|
|
// Now shut down for good and wait for the kubelet to react.
|
|
// This takes time...
|
|
ginkgo.By("uninstalling driver and waiting for ResourceSlice wiping")
|
|
newDriver.TearDown(tCtx)
|
|
newDriver.IsGone(tCtx)
|
|
})
|
|
})
|
|
|
|
// Tests that have the `withKubelet` argument can run with or without kubelet support for plugins and DRA, aka feature.DynamicResourceAllocation.
|
|
// Without it, the test driver publishes ResourceSlices, but does not attempt to register itself.
|
|
// Tests only expect pods to get scheduled, but not to become running.
|
|
//
|
|
// TODO before conformance promotion: add https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md#sample-conformance-test meta data
|
|
|
|
singleNodeTests := func(withKubelet bool) {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
maxAllocations := 1
|
|
numPods := 5
|
|
driver := drautils.NewDriver(f, nodes, drautils.DriverResources(maxAllocations)) // All tests get their own driver instance.
|
|
driver.WithKubelet = withKubelet
|
|
b := drautils.NewBuilder(f, driver)
|
|
// We have to set the parameters *before* creating the class.
|
|
b.ClassParameters = `{"x":"y"}`
|
|
expectedEnv := []string{"admin_x", "y"}
|
|
_, expected := b.ParametersEnv()
|
|
expectedEnv = append(expectedEnv, expected...)
|
|
|
|
ginkgo.It("supports claim and class parameters", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
b.Create(tCtx, pod, template)
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
})
|
|
|
|
ginkgo.It("supports reusing resources", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objects []klog.KMetadata
|
|
pods := make([]*v1.Pod, numPods)
|
|
for i := 0; i < numPods; i++ {
|
|
pod, template := b.PodInline()
|
|
pods[i] = pod
|
|
objects = append(objects, pod, template)
|
|
}
|
|
|
|
b.Create(tCtx, objects...)
|
|
|
|
// We don't know the order. All that matters is that all of them get scheduled eventually.
|
|
var wg sync.WaitGroup
|
|
wg.Add(numPods)
|
|
for i := 0; i < numPods; i++ {
|
|
pod := pods[i]
|
|
go func() {
|
|
defer ginkgo.GinkgoRecover()
|
|
defer wg.Done()
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
err := f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{})
|
|
framework.ExpectNoError(err, "delete pod")
|
|
framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, time.Duration(numPods)*f.Timeouts.PodStartSlow))
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
})
|
|
|
|
ginkgo.It("supports sharing a claim concurrently", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objects []klog.KMetadata
|
|
objects = append(objects, b.ExternalClaim())
|
|
pods := make([]*v1.Pod, numPods)
|
|
for i := 0; i < numPods; i++ {
|
|
pod := b.PodExternal()
|
|
pods[i] = pod
|
|
objects = append(objects, pod)
|
|
}
|
|
|
|
b.Create(tCtx, objects...)
|
|
|
|
// We don't know the order. All that matters is that all of them get scheduled eventually.
|
|
f.Timeouts.PodStartSlow *= time.Duration(numPods)
|
|
var wg sync.WaitGroup
|
|
wg.Add(numPods)
|
|
for i := 0; i < numPods; i++ {
|
|
pod := pods[i]
|
|
go func() {
|
|
defer ginkgo.GinkgoRecover()
|
|
defer wg.Done()
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
})
|
|
|
|
ginkgo.It("retries pod scheduling after creating device class", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objects []klog.KMetadata
|
|
pod, template := b.PodInline()
|
|
deviceClassName := template.Spec.Spec.Devices.Requests[0].Exactly.DeviceClassName
|
|
class, err := f.ClientSet.ResourceV1().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
deviceClassName += "-b"
|
|
template.Spec.Spec.Devices.Requests[0].Exactly.DeviceClassName = deviceClassName
|
|
objects = append(objects, template, pod)
|
|
b.Create(tCtx, objects...)
|
|
|
|
framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
|
|
|
|
class.UID = ""
|
|
class.ResourceVersion = ""
|
|
class.Name = deviceClassName
|
|
b.Create(tCtx, class)
|
|
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
})
|
|
|
|
ginkgo.It("retries pod scheduling after updating device class", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objects []klog.KMetadata
|
|
pod, template := b.PodInline()
|
|
|
|
// First modify the class so that it matches no nodes (for classic DRA) and no devices (structured parameters).
|
|
deviceClassName := template.Spec.Spec.Devices.Requests[0].Exactly.DeviceClassName
|
|
class, err := f.ClientSet.ResourceV1().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
originalClass := class.DeepCopy()
|
|
class.Spec.Selectors = []resourceapi.DeviceSelector{{
|
|
CEL: &resourceapi.CELDeviceSelector{
|
|
Expression: "false",
|
|
},
|
|
}}
|
|
class, err = f.ClientSet.ResourceV1().DeviceClasses().Update(ctx, class, metav1.UpdateOptions{})
|
|
framework.ExpectNoError(err)
|
|
|
|
// Now create the pod.
|
|
objects = append(objects, template, pod)
|
|
b.Create(tCtx, objects...)
|
|
|
|
framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
|
|
|
|
// Unblock the pod.
|
|
class.Spec.Selectors = originalClass.Spec.Selectors
|
|
_, err = f.ClientSet.ResourceV1().DeviceClasses().Update(ctx, class, metav1.UpdateOptions{})
|
|
framework.ExpectNoError(err)
|
|
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
})
|
|
|
|
ginkgo.It("runs a pod without a generated resource claim", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, _ /* template */ := b.PodInline()
|
|
created := b.Create(tCtx, pod)
|
|
pod = created[0].(*v1.Pod)
|
|
|
|
// Normally, this pod would be stuck because the
|
|
// ResourceClaim cannot be created without the
|
|
// template. We allow it to run by communicating
|
|
// through the status that the ResourceClaim is not
|
|
// needed.
|
|
pod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{
|
|
{Name: pod.Spec.ResourceClaims[0].Name, ResourceClaimName: nil},
|
|
}
|
|
_, err := f.ClientSet.CoreV1().Pods(pod.Namespace).UpdateStatus(ctx, pod, metav1.UpdateOptions{})
|
|
framework.ExpectNoError(err)
|
|
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod))
|
|
})
|
|
|
|
ginkgo.It("supports simple pod referencing inline resource claim", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
b.Create(tCtx, pod, template)
|
|
b.TestPod(tCtx, pod)
|
|
})
|
|
|
|
ginkgo.It("supports inline claim referenced by multiple containers", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInlineMultiple()
|
|
b.Create(tCtx, pod, template)
|
|
b.TestPod(tCtx, pod)
|
|
})
|
|
|
|
ginkgo.It("supports simple pod referencing external resource claim", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod := b.PodExternal()
|
|
claim := b.ExternalClaim()
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod)
|
|
})
|
|
|
|
ginkgo.It("supports external claim referenced by multiple pods", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod1 := b.PodExternal()
|
|
pod2 := b.PodExternal()
|
|
pod3 := b.PodExternal()
|
|
claim := b.ExternalClaim()
|
|
b.Create(tCtx, claim, pod1, pod2, pod3)
|
|
|
|
for _, pod := range []*v1.Pod{pod1, pod2, pod3} {
|
|
b.TestPod(tCtx, pod)
|
|
}
|
|
})
|
|
|
|
ginkgo.It("supports external claim referenced by multiple containers of multiple pods", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod1 := b.PodExternalMultiple()
|
|
pod2 := b.PodExternalMultiple()
|
|
pod3 := b.PodExternalMultiple()
|
|
claim := b.ExternalClaim()
|
|
b.Create(tCtx, claim, pod1, pod2, pod3)
|
|
|
|
for _, pod := range []*v1.Pod{pod1, pod2, pod3} {
|
|
b.TestPod(tCtx, pod)
|
|
}
|
|
})
|
|
|
|
ginkgo.It("supports init containers", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]}
|
|
pod.Spec.InitContainers[0].Name += "-init"
|
|
// This must succeed for the pod to start.
|
|
pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env | grep user_a=b"}
|
|
b.Create(tCtx, pod, template)
|
|
|
|
b.TestPod(tCtx, pod)
|
|
})
|
|
|
|
ginkgo.It("must deallocate after use", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod := b.PodExternal()
|
|
claim := b.ExternalClaim()
|
|
b.Create(tCtx, claim, pod)
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
|
return f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
}).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil)))
|
|
|
|
b.TestPod(tCtx, pod)
|
|
|
|
ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(pod)))
|
|
framework.ExpectNoError(f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{}))
|
|
|
|
ginkgo.By("waiting for claim to get deallocated")
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
|
return f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil)))
|
|
})
|
|
|
|
if withKubelet {
|
|
// Serial because the example device plugin can only be deployed with one instance at a time.
|
|
f.It("supports extended resources together with ResourceClaim", f.WithSerial(), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
extendedResourceName := deployDevicePlugin(ctx, f, nodes.NodeNames[0:1])
|
|
|
|
pod := b.PodExternal()
|
|
resources := v1.ResourceList{extendedResourceName: resource.MustParse("1")}
|
|
pod.Spec.Containers[0].Resources.Requests = resources
|
|
pod.Spec.Containers[0].Resources.Limits = resources
|
|
claim := b.ExternalClaim()
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod)
|
|
})
|
|
}
|
|
}
|
|
|
|
// The following tests only make sense when there is more than one node.
|
|
// They get skipped when there's only one node.
|
|
multiNodeTests := func(withKubelet bool) {
|
|
nodes := drautils.NewNodes(f, 2, 8)
|
|
|
|
ginkgo.Context("with different ResourceSlices", func() {
|
|
firstDevice := "pre-defined-device-01"
|
|
secondDevice := "pre-defined-device-02"
|
|
devicesPerNode := []map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{
|
|
// First node:
|
|
{
|
|
firstDevice: {
|
|
"healthy": {BoolValue: ptr.To(true)},
|
|
"exists": {BoolValue: ptr.To(true)},
|
|
},
|
|
},
|
|
// Second node:
|
|
{
|
|
secondDevice: {
|
|
"healthy": {BoolValue: ptr.To(false)},
|
|
// Has no "exists" attribute!
|
|
},
|
|
},
|
|
}
|
|
driver := drautils.NewDriver(f, nodes, drautils.DriverResources(-1, devicesPerNode...))
|
|
driver.WithKubelet = withKubelet
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
ginkgo.It("keeps pod pending because of CEL runtime errors", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
|
|
// When pod scheduling encounters CEL runtime errors for some nodes, but not all,
|
|
// it should still not schedule the pod because there is something wrong with it.
|
|
// Scheduling it would make it harder to detect that there is a problem.
|
|
//
|
|
// This matches the "CEL-runtime-error-for-subset-of-nodes" unit test, except that
|
|
// here we try it in combination with the actual scheduler and can extend it with
|
|
// other checks, like event handling (future extension).
|
|
|
|
gomega.Eventually(ctx, framework.ListObjects(f.ClientSet.ResourceV1().ResourceSlices().List,
|
|
metav1.ListOptions{
|
|
FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driver.Name,
|
|
},
|
|
)).Should(gomega.HaveField("Items", gomega.ConsistOf(
|
|
gomega.HaveField("Spec.Devices", gomega.ConsistOf(
|
|
gomega.Equal(resourceapi.Device{
|
|
Name: firstDevice,
|
|
Attributes: devicesPerNode[0][firstDevice],
|
|
}))),
|
|
gomega.HaveField("Spec.Devices", gomega.ConsistOf(
|
|
gomega.Equal(resourceapi.Device{
|
|
Name: secondDevice,
|
|
Attributes: devicesPerNode[1][secondDevice],
|
|
}))),
|
|
)))
|
|
|
|
pod, template := b.PodInline()
|
|
template.Spec.Spec.Devices.Requests[0].Exactly.Selectors = append(template.Spec.Spec.Devices.Requests[0].Exactly.Selectors,
|
|
resourceapi.DeviceSelector{
|
|
CEL: &resourceapi.CELDeviceSelector{
|
|
// Runtime error on one node, but not all.
|
|
Expression: fmt.Sprintf(`device.attributes["%s"].exists`, driver.Name),
|
|
},
|
|
},
|
|
)
|
|
b.Create(tCtx, pod, template)
|
|
|
|
framework.ExpectNoError(e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "scheduling failure", f.Timeouts.PodStartShort, func(pod *v1.Pod) (bool, error) {
|
|
for _, condition := range pod.Status.Conditions {
|
|
if condition.Type == "PodScheduled" {
|
|
if condition.Status != "False" {
|
|
gomega.StopTrying("pod got scheduled unexpectedly").Now()
|
|
}
|
|
if strings.Contains(condition.Message, "CEL runtime error") {
|
|
// This is what we are waiting for.
|
|
return true, nil
|
|
}
|
|
}
|
|
}
|
|
return false, nil
|
|
}), "pod must not get scheduled because of a CEL runtime error")
|
|
})
|
|
})
|
|
|
|
ginkgo.Context("with node-local resources", func() {
|
|
driver := drautils.NewDriver(f, nodes, drautils.DriverResources(1))
|
|
driver.WithKubelet = withKubelet
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
ginkgo.It("uses all resources", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objs []klog.KMetadata
|
|
var pods []*v1.Pod
|
|
for i := 0; i < len(nodes.NodeNames); i++ {
|
|
pod, template := b.PodInline()
|
|
pods = append(pods, pod)
|
|
objs = append(objs, pod, template)
|
|
}
|
|
b.Create(tCtx, objs...)
|
|
|
|
for _, pod := range pods {
|
|
if withKubelet {
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
} else {
|
|
err := e2epod.WaitForPodScheduled(ctx, f.ClientSet, pod.Namespace, pod.Name)
|
|
framework.ExpectNoError(err, "schedule pod")
|
|
}
|
|
}
|
|
|
|
// The pods all should run on different
|
|
// nodes because the maximum number of
|
|
// claims per node was limited to 1 for
|
|
// this test.
|
|
//
|
|
// We cannot know for sure why the pods
|
|
// ran on two different nodes (could
|
|
// also be a coincidence) but if they
|
|
// don't cover all nodes, then we have
|
|
// a problem.
|
|
used := make(map[string]*v1.Pod)
|
|
for _, pod := range pods {
|
|
pod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
framework.ExpectNoError(err, "get pod")
|
|
nodeName := pod.Spec.NodeName
|
|
if other, ok := used[nodeName]; ok {
|
|
framework.Failf("Pod %s got started on the same node %s as pod %s although claim allocation should have been limited to one claim per node.", pod.Name, nodeName, other.Name)
|
|
}
|
|
used[nodeName] = pod
|
|
}
|
|
})
|
|
})
|
|
}
|
|
|
|
prioritizedListTests := func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
|
|
driver1Params, driver1Env := `{"driver":"1"}`, []string{"admin_driver", "1"}
|
|
driver2Params, driver2Env := `{"driver":"2"}`, []string{"admin_driver", "2"}
|
|
|
|
driver1 := drautils.NewDriver(f, nodes, drautils.DriverResources(-1, []map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{
|
|
{
|
|
"device-1-1": {
|
|
"dra.example.com/version": {StringValue: ptr.To("1.0.0")},
|
|
"dra.example.com/pcieRoot": {StringValue: ptr.To("bar")},
|
|
},
|
|
"device-1-2": {
|
|
"dra.example.com/version": {StringValue: ptr.To("2.0.0")},
|
|
"dra.example.com/pcieRoot": {StringValue: ptr.To("foo")},
|
|
},
|
|
},
|
|
}...))
|
|
driver1.NameSuffix = "-1"
|
|
b1 := drautils.NewBuilder(f, driver1)
|
|
b1.ClassParameters = driver1Params
|
|
|
|
driver2 := drautils.NewDriver(f, nodes, drautils.DriverResources(-1, []map[string]map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{
|
|
{
|
|
"device-2-1": {
|
|
"dra.example.com/version": {StringValue: ptr.To("1.0.0")},
|
|
"dra.example.com/pcieRoot": {StringValue: ptr.To("foo")},
|
|
},
|
|
},
|
|
}...))
|
|
driver2.NameSuffix = "-2"
|
|
b2 := drautils.NewBuilder(f, driver2)
|
|
b2.ClassParameters = driver2Params
|
|
|
|
f.It("selects the first subrequest that can be satisfied", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
name := "external-multiclaim"
|
|
params := `{"a":"b"}`
|
|
claim := &resourceapi.ResourceClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Spec: resourceapi.ResourceClaimSpec{
|
|
Devices: resourceapi.DeviceClaim{
|
|
Requests: []resourceapi.DeviceRequest{{
|
|
Name: "request-1",
|
|
FirstAvailable: []resourceapi.DeviceSubRequest{
|
|
{
|
|
Name: "sub-request-1",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 3,
|
|
},
|
|
{
|
|
Name: "sub-request-2",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 2,
|
|
},
|
|
{
|
|
Name: "sub-request-3",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 1,
|
|
},
|
|
},
|
|
}},
|
|
Config: []resourceapi.DeviceClaimConfiguration{
|
|
{
|
|
Requests: []string{"request-1"},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver1.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(params),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
pod := b1.PodExternal()
|
|
podClaimName := "resource-claim"
|
|
externalClaimName := "external-multiclaim"
|
|
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
|
{
|
|
Name: podClaimName,
|
|
ResourceClaimName: &externalClaimName,
|
|
},
|
|
}
|
|
b1.Create(tCtx, claim, pod)
|
|
b1.TestPod(tCtx, pod)
|
|
|
|
var allocatedResourceClaim *resourceapi.ResourceClaim
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
|
var err error
|
|
allocatedResourceClaim, err = f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
return allocatedResourceClaim, err
|
|
}).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil)))
|
|
results := allocatedResourceClaim.Status.Allocation.Devices.Results
|
|
gomega.Expect(results).To(gomega.HaveLen(2))
|
|
gomega.Expect(results[0].Request).To(gomega.Equal("request-1/sub-request-2"))
|
|
gomega.Expect(results[1].Request).To(gomega.Equal("request-1/sub-request-2"))
|
|
})
|
|
|
|
f.It("uses the config for the selected subrequest", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
name := "external-multiclaim"
|
|
parentReqParams, parentReqEnv := `{"a":"b"}`, []string{"user_a", "b"}
|
|
subReq1Params := `{"c":"d"}`
|
|
subReq2Params, subReq2Env := `{"e":"f"}`, []string{"user_e", "f"}
|
|
claim := &resourceapi.ResourceClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Spec: resourceapi.ResourceClaimSpec{
|
|
Devices: resourceapi.DeviceClaim{
|
|
Requests: []resourceapi.DeviceRequest{{
|
|
Name: "request-1",
|
|
FirstAvailable: []resourceapi.DeviceSubRequest{
|
|
{
|
|
Name: "sub-request-1",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 3,
|
|
},
|
|
{
|
|
Name: "sub-request-2",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 2,
|
|
},
|
|
},
|
|
}},
|
|
Config: []resourceapi.DeviceClaimConfiguration{
|
|
{
|
|
Requests: []string{"request-1"},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver1.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(parentReqParams),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Requests: []string{"request-1/sub-request-1"},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver1.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(subReq1Params),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Requests: []string{"request-1/sub-request-2"},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver1.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(subReq2Params),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
pod := b1.PodExternal()
|
|
podClaimName := "resource-claim"
|
|
externalClaimName := "external-multiclaim"
|
|
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
|
{
|
|
Name: podClaimName,
|
|
ResourceClaimName: &externalClaimName,
|
|
},
|
|
}
|
|
b1.Create(tCtx, claim, pod)
|
|
var expectedEnv []string
|
|
expectedEnv = append(expectedEnv, parentReqEnv...)
|
|
expectedEnv = append(expectedEnv, subReq2Env...)
|
|
b1.TestPod(tCtx, pod, expectedEnv...)
|
|
})
|
|
|
|
f.It("chooses the correct subrequest subject to constraints", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
name := "external-multiclaim"
|
|
params := `{"a":"b"}`
|
|
claim := &resourceapi.ResourceClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Spec: resourceapi.ResourceClaimSpec{
|
|
Devices: resourceapi.DeviceClaim{
|
|
Requests: []resourceapi.DeviceRequest{
|
|
{
|
|
Name: "request-1",
|
|
FirstAvailable: []resourceapi.DeviceSubRequest{
|
|
{
|
|
Name: "sub-request-1",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 1,
|
|
},
|
|
{
|
|
Name: "sub-request-2",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 1,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "request-2",
|
|
Exactly: &resourceapi.ExactDeviceRequest{
|
|
DeviceClassName: b2.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 1,
|
|
},
|
|
},
|
|
},
|
|
Constraints: []resourceapi.DeviceConstraint{
|
|
{
|
|
Requests: []string{"request-1", "request-2"},
|
|
MatchAttribute: ptr.To(resourceapi.FullyQualifiedName("dra.example.com/version")),
|
|
},
|
|
{
|
|
Requests: []string{"request-1/sub-request-1", "request-2"},
|
|
MatchAttribute: ptr.To(resourceapi.FullyQualifiedName("dra.example.com/pcieRoot")),
|
|
},
|
|
},
|
|
Config: []resourceapi.DeviceClaimConfiguration{
|
|
{
|
|
Requests: []string{},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver1.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(params),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
pod := b1.PodExternal()
|
|
podClaimName := "resource-claim"
|
|
externalClaimName := "external-multiclaim"
|
|
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
|
{
|
|
Name: podClaimName,
|
|
ResourceClaimName: &externalClaimName,
|
|
},
|
|
}
|
|
b1.Create(tCtx, claim, pod)
|
|
b1.TestPod(tCtx, pod)
|
|
|
|
var allocatedResourceClaim *resourceapi.ResourceClaim
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
|
var err error
|
|
allocatedResourceClaim, err = f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
return allocatedResourceClaim, err
|
|
}).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil)))
|
|
results := allocatedResourceClaim.Status.Allocation.Devices.Results
|
|
gomega.Expect(results).To(gomega.HaveLen(2))
|
|
gomega.Expect(results[0].Request).To(gomega.Equal("request-1/sub-request-2"))
|
|
gomega.Expect(results[1].Request).To(gomega.Equal("request-2"))
|
|
})
|
|
|
|
f.It("filters config correctly for multiple devices", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
name := "external-multiclaim"
|
|
req1Params, req1Env := `{"a":"b"}`, []string{"user_a", "b"}
|
|
req1subReq1Params, _ := `{"c":"d"}`, []string{"user_d", "d"}
|
|
req1subReq2Params, req1subReq2Env := `{"e":"f"}`, []string{"user_e", "f"}
|
|
req2Params, req2Env := `{"g":"h"}`, []string{"user_g", "h"}
|
|
claim := &resourceapi.ResourceClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Spec: resourceapi.ResourceClaimSpec{
|
|
Devices: resourceapi.DeviceClaim{
|
|
Requests: []resourceapi.DeviceRequest{
|
|
{
|
|
Name: "request-1",
|
|
FirstAvailable: []resourceapi.DeviceSubRequest{
|
|
{
|
|
Name: "sub-request-1",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 20, // Requests more than are available.
|
|
},
|
|
{
|
|
Name: "sub-request-2",
|
|
DeviceClassName: b1.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 1,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "request-2",
|
|
Exactly: &resourceapi.ExactDeviceRequest{
|
|
DeviceClassName: b2.ClassName(),
|
|
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
|
|
Count: 1,
|
|
},
|
|
},
|
|
},
|
|
Config: []resourceapi.DeviceClaimConfiguration{
|
|
{
|
|
Requests: []string{"request-1"},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver1.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(req1Params),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Requests: []string{"request-1/sub-request-1"},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver1.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(req1subReq1Params),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Requests: []string{"request-1/sub-request-2"},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver1.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(req1subReq2Params),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Requests: []string{"request-2"},
|
|
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
|
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
|
Driver: driver2.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(req2Params),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
pod := b1.Pod()
|
|
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
|
|
pod.Spec.Containers[0].Name = "with-resource-0"
|
|
pod.Spec.Containers[1].Name = "with-resource-1"
|
|
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
|
{
|
|
Name: name,
|
|
ResourceClaimName: &name,
|
|
},
|
|
}
|
|
pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: name, Request: "request-1"}}
|
|
pod.Spec.Containers[1].Resources.Claims = []v1.ResourceClaim{{Name: name, Request: "request-2"}}
|
|
|
|
b1.Create(tCtx, claim, pod)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
|
|
var allocatedResourceClaim *resourceapi.ResourceClaim
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
|
var err error
|
|
allocatedResourceClaim, err = f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
return allocatedResourceClaim, err
|
|
}).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil)))
|
|
results := allocatedResourceClaim.Status.Allocation.Devices.Results
|
|
gomega.Expect(results).To(gomega.HaveLen(2))
|
|
gomega.Expect(results[0].Request).To(gomega.Equal("request-1/sub-request-2"))
|
|
gomega.Expect(results[1].Request).To(gomega.Equal("request-2"))
|
|
|
|
req1ExpectedEnv := []string{
|
|
"claim_external_multiclaim_request_1",
|
|
"true",
|
|
}
|
|
req1ExpectedEnv = append(req1ExpectedEnv, req1Env...)
|
|
req1ExpectedEnv = append(req1ExpectedEnv, req1subReq2Env...)
|
|
req1ExpectedEnv = append(req1ExpectedEnv, driver1Env...)
|
|
drautils.TestContainerEnv(tCtx, pod, "with-resource-0", true, req1ExpectedEnv...)
|
|
|
|
req2ExpectedEnv := []string{
|
|
"claim_external_multiclaim_request_2",
|
|
"true",
|
|
}
|
|
req2ExpectedEnv = append(req2ExpectedEnv, req2Env...)
|
|
req2ExpectedEnv = append(req2ExpectedEnv, driver2Env...)
|
|
drautils.TestContainerEnv(tCtx, pod, "with-resource-1", true, req2ExpectedEnv...)
|
|
})
|
|
}
|
|
|
|
v1beta1Tests := func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
maxAllocations := 1
|
|
driver := drautils.NewDriver(f, nodes, drautils.DriverResources(maxAllocations))
|
|
b := drautils.NewBuilder(f, driver)
|
|
// We have to set the parameters *before* creating the class.
|
|
b.ClassParameters = `{"x":"y"}`
|
|
expectedEnv := []string{"admin_x", "y"}
|
|
_, expected := b.ParametersEnv()
|
|
expectedEnv = append(expectedEnv, expected...)
|
|
|
|
ginkgo.It("supports simple ResourceClaim", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInlineWithV1beta1()
|
|
b.Create(tCtx, pod, template)
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
})
|
|
|
|
f.It("supports requests with alternatives", f.WithFeatureGate(features.DRAPrioritizedList), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
claimName := "external-multiclaim"
|
|
parameters, _ := b.ParametersEnv()
|
|
claim := &resourcev1beta1.ResourceClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: claimName,
|
|
},
|
|
Spec: resourcev1beta1.ResourceClaimSpec{
|
|
Devices: resourcev1beta1.DeviceClaim{
|
|
Requests: []resourcev1beta1.DeviceRequest{{
|
|
Name: "request-1",
|
|
FirstAvailable: []resourcev1beta1.DeviceSubRequest{
|
|
{
|
|
Name: "sub-request-1",
|
|
DeviceClassName: b.ClassName(),
|
|
AllocationMode: resourcev1beta1.DeviceAllocationModeExactCount,
|
|
Count: 2,
|
|
},
|
|
{
|
|
Name: "sub-request-2",
|
|
DeviceClassName: b.ClassName(),
|
|
AllocationMode: resourcev1beta1.DeviceAllocationModeExactCount,
|
|
Count: 1,
|
|
},
|
|
},
|
|
}},
|
|
Config: []resourcev1beta1.DeviceClaimConfiguration{{
|
|
DeviceConfiguration: resourcev1beta1.DeviceConfiguration{
|
|
Opaque: &resourcev1beta1.OpaqueDeviceConfiguration{
|
|
Driver: driver.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(parameters),
|
|
},
|
|
},
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
}
|
|
pod := b.PodExternal()
|
|
podClaimName := "resource-claim"
|
|
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
|
{
|
|
Name: podClaimName,
|
|
ResourceClaimName: &claimName,
|
|
},
|
|
}
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
|
|
var allocatedResourceClaim *resourcev1beta1.ResourceClaim
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1beta1.ResourceClaim, error) {
|
|
var err error
|
|
allocatedResourceClaim, err = f.ClientSet.ResourceV1beta1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
return allocatedResourceClaim, err
|
|
}).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourcev1beta1.AllocationResult)(nil)))
|
|
results := allocatedResourceClaim.Status.Allocation.Devices.Results
|
|
gomega.Expect(results).To(gomega.HaveLen(1))
|
|
gomega.Expect(results[0].Request).To(gomega.Equal("request-1/sub-request-2"))
|
|
})
|
|
}
|
|
|
|
v1beta2Tests := func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
maxAllocations := 1
|
|
driver := drautils.NewDriver(f, nodes, drautils.DriverResources(maxAllocations))
|
|
b := drautils.NewBuilder(f, driver)
|
|
// We have to set the parameters *before* creating the class.
|
|
b.ClassParameters = `{"x":"y"}`
|
|
expectedEnv := []string{"admin_x", "y"}
|
|
_, expected := b.ParametersEnv()
|
|
expectedEnv = append(expectedEnv, expected...)
|
|
|
|
ginkgo.It("supports simple ResourceClaim", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInlineWithV1beta2()
|
|
b.Create(tCtx, pod, template)
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
})
|
|
|
|
f.It("supports requests with alternatives", f.WithFeatureGate(features.DRAPrioritizedList), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
claimName := "external-multiclaim"
|
|
parameters, _ := b.ParametersEnv()
|
|
claim := &resourcev1beta2.ResourceClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: claimName,
|
|
},
|
|
Spec: resourcev1beta2.ResourceClaimSpec{
|
|
Devices: resourcev1beta2.DeviceClaim{
|
|
Requests: []resourcev1beta2.DeviceRequest{{
|
|
Name: "request-1",
|
|
FirstAvailable: []resourcev1beta2.DeviceSubRequest{
|
|
{
|
|
Name: "sub-request-1",
|
|
DeviceClassName: b.ClassName(),
|
|
AllocationMode: resourcev1beta2.DeviceAllocationModeExactCount,
|
|
Count: 2,
|
|
},
|
|
{
|
|
Name: "sub-request-2",
|
|
DeviceClassName: b.ClassName(),
|
|
AllocationMode: resourcev1beta2.DeviceAllocationModeExactCount,
|
|
Count: 1,
|
|
},
|
|
},
|
|
}},
|
|
Config: []resourcev1beta2.DeviceClaimConfiguration{{
|
|
DeviceConfiguration: resourcev1beta2.DeviceConfiguration{
|
|
Opaque: &resourcev1beta2.OpaqueDeviceConfiguration{
|
|
Driver: driver.Name,
|
|
Parameters: runtime.RawExtension{
|
|
Raw: []byte(parameters),
|
|
},
|
|
},
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
}
|
|
pod := b.PodExternal()
|
|
podClaimName := "resource-claim"
|
|
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
|
{
|
|
Name: podClaimName,
|
|
ResourceClaimName: &claimName,
|
|
},
|
|
}
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod, expectedEnv...)
|
|
|
|
var allocatedResourceClaim *resourcev1beta2.ResourceClaim
|
|
gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1beta2.ResourceClaim, error) {
|
|
var err error
|
|
allocatedResourceClaim, err = f.ClientSet.ResourceV1beta2().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
return allocatedResourceClaim, err
|
|
}).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourcev1beta2.AllocationResult)(nil)))
|
|
results := allocatedResourceClaim.Status.Allocation.Devices.Results
|
|
gomega.Expect(results).To(gomega.HaveLen(1))
|
|
gomega.Expect(results[0].Request).To(gomega.Equal("request-1/sub-request-2"))
|
|
})
|
|
}
|
|
|
|
partitionableDevicesTests := func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
driver := drautils.NewDriver(f, nodes, drautils.ToDriverResources(
|
|
[]resourceapi.CounterSet{
|
|
{
|
|
Name: "counter-1",
|
|
Counters: map[string]resourceapi.Counter{
|
|
"memory": {
|
|
Value: resource.MustParse("6Gi"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
[]resourceapi.Device{
|
|
{
|
|
Name: "device-1",
|
|
ConsumesCounters: []resourceapi.DeviceCounterConsumption{
|
|
{
|
|
CounterSet: "counter-1",
|
|
Counters: map[string]resourceapi.Counter{
|
|
"memory": {
|
|
Value: resource.MustParse("4Gi"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "device-2",
|
|
ConsumesCounters: []resourceapi.DeviceCounterConsumption{
|
|
{
|
|
CounterSet: "counter-1",
|
|
Counters: map[string]resourceapi.Counter{
|
|
"memory": {
|
|
Value: resource.MustParse("4Gi"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}...,
|
|
))
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
f.It("must consume and free up counters", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
// The first pod will use one of the devices. Since both devices are
|
|
// available, there should be sufficient counters left to allocate
|
|
// a device.
|
|
claim := b.ExternalClaim()
|
|
pod := b.PodExternal()
|
|
pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod)
|
|
|
|
// For the second pod, there should not be sufficient counters left, so
|
|
// it should not succeed. This means the pod should remain in the pending state.
|
|
claim2 := b.ExternalClaim()
|
|
pod2 := b.PodExternal()
|
|
pod2.Spec.ResourceClaims[0].ResourceClaimName = &claim2.Name
|
|
b.Create(tCtx, claim2, pod2)
|
|
|
|
gomega.Consistently(ctx, func(ctx context.Context) error {
|
|
testPod, err := f.ClientSet.CoreV1().Pods(pod2.Namespace).Get(ctx, pod2.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("expected the test pod %s to exist: %w", pod2.Name, err)
|
|
}
|
|
if testPod.Status.Phase != v1.PodPending {
|
|
return fmt.Errorf("pod %s: unexpected status %s, expected status: %s", pod2.Name, testPod.Status.Phase, v1.PodPending)
|
|
}
|
|
return nil
|
|
}, 20*time.Second, 200*time.Millisecond).Should(gomega.Succeed())
|
|
|
|
// Delete the first pod
|
|
b.DeletePodAndWaitForNotFound(tCtx, pod)
|
|
|
|
// There shoud not be available devices for pod2.
|
|
b.TestPod(tCtx, pod2)
|
|
})
|
|
}
|
|
|
|
consumableCapacityTests := func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
// single device which allows multiple allocations and has 80Gi consumable memory.
|
|
driver := drautils.NewDriver(f, nodes, drautils.ToDriverResources(
|
|
[]resourceapi.CounterSet{},
|
|
[]resourceapi.Device{
|
|
{
|
|
Name: "consumable-device-1",
|
|
AllowMultipleAllocations: ptr.To(true),
|
|
Capacity: map[resourceapi.QualifiedName]resourceapi.DeviceCapacity{
|
|
"memory": {
|
|
Value: resource.MustParse("8Gi"),
|
|
RequestPolicy: &resourceapi.CapacityRequestPolicy{
|
|
Default: ptr.To(resource.MustParse("1Gi")),
|
|
ValidRange: &resourceapi.CapacityRequestPolicyRange{
|
|
Min: ptr.To(resource.MustParse("1Gi")),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}...,
|
|
))
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
f.It("must allow multiple allocations and consume capacity", f.WithLabel("KubeletMinVersion:1.34"), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
// The first pod will use 4Gi of the device.
|
|
claim := b.ExternalClaim()
|
|
claim.Spec.Devices.Requests[0].Exactly.Capacity = &resourceapi.CapacityRequirements{
|
|
Requests: map[resourceapi.QualifiedName]resource.Quantity{
|
|
"memory": resource.MustParse("4Gi"),
|
|
},
|
|
}
|
|
pod := b.PodExternal()
|
|
pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name
|
|
b.Create(tCtx, claim, pod)
|
|
b.TestPod(tCtx, pod)
|
|
|
|
// The second pod will be failed to request 8Gi capacity.
|
|
claim2 := b.ExternalClaim()
|
|
claim2.Spec.Devices.Requests[0].Exactly.Capacity = &resourceapi.CapacityRequirements{
|
|
Requests: map[resourceapi.QualifiedName]resource.Quantity{
|
|
"memory": resource.MustParse("8Gi"),
|
|
},
|
|
}
|
|
pod2 := b.PodExternal()
|
|
pod2.Spec.ResourceClaims[0].ResourceClaimName = &claim2.Name
|
|
b.Create(tCtx, claim2, pod2)
|
|
|
|
// The third pod should be able to use the rest 4Gi of the device.
|
|
claim3 := b.ExternalClaim()
|
|
claim3.Spec.Devices.Requests[0].Exactly.Capacity = &resourceapi.CapacityRequirements{
|
|
Requests: map[resourceapi.QualifiedName]resource.Quantity{
|
|
"memory": resource.MustParse("4Gi"),
|
|
},
|
|
}
|
|
pod3 := b.PodExternal()
|
|
pod3.Spec.ResourceClaims[0].ResourceClaimName = &claim3.Name
|
|
b.Create(tCtx, claim3, pod3)
|
|
b.TestPod(tCtx, pod3)
|
|
|
|
gomega.Consistently(ctx, func(ctx context.Context) error {
|
|
testPod2, err := f.ClientSet.CoreV1().Pods(pod2.Namespace).Get(ctx, pod2.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("expected the test pod %s to exist: %w", pod2.Name, err)
|
|
}
|
|
if testPod2.Status.Phase != v1.PodPending {
|
|
return fmt.Errorf("pod %s: unexpected status %s, expected status: %s", pod2.Name, testPod2.Status.Phase, v1.PodPending)
|
|
}
|
|
return nil
|
|
}, 20*time.Second, 200*time.Millisecond).Should(gomega.Succeed())
|
|
|
|
// Delete the first and third pod
|
|
b.DeletePodAndWaitForNotFound(tCtx, pod)
|
|
b.DeletePodAndWaitForNotFound(tCtx, pod3)
|
|
|
|
// There should be available capacity for pod2 now.
|
|
b.TestPod(tCtx, pod2)
|
|
})
|
|
}
|
|
|
|
// It is okay to use the same context multiple times (like "control plane"),
|
|
// as long as the test names the still remain unique overall.
|
|
|
|
framework.Context("control plane", func() { singleNodeTests(false) })
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, "on single node", func() { singleNodeTests(true) })
|
|
|
|
framework.Context("control plane", func() { multiNodeTests(false) })
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, "on multiple nodes", func() { multiNodeTests(true) })
|
|
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, f.WithFeatureGate(features.DRAPrioritizedList), prioritizedListTests)
|
|
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, f.WithFeatureGate(features.DRAConsumableCapacity), consumableCapacityTests)
|
|
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, "with v1beta1 API", v1beta1Tests)
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, "with v1beta2 API", v1beta2Tests)
|
|
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, f.WithFeatureGate(features.DRAPartitionableDevices), partitionableDevicesTests)
|
|
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, f.WithFeatureGate(features.DRADeviceTaints), func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false), drautils.TaintAllDevices(resourceapi.DeviceTaint{
|
|
Key: "example.com/taint",
|
|
Value: "tainted",
|
|
Effect: resourceapi.DeviceTaintEffectNoSchedule,
|
|
}))
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
f.It("NoSchedule keeps pod pending", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
b.Create(tCtx, pod, template)
|
|
framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name))
|
|
})
|
|
|
|
f.It("NoSchedule can be tolerated", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
template.Spec.Spec.Devices.Requests[0].Exactly.Tolerations = []resourceapi.DeviceToleration{{
|
|
Effect: resourceapi.DeviceTaintEffectNoSchedule,
|
|
Operator: resourceapi.DeviceTolerationOpExists,
|
|
// No key: tolerate *all* taints with this effect.
|
|
}}
|
|
b.Create(tCtx, pod, template)
|
|
b.TestPod(tCtx, pod)
|
|
})
|
|
|
|
f.It("DeviceTaintRule evicts pod", f.WithFeatureGate(features.DRADeviceTaintRules), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
template.Spec.Spec.Devices.Requests[0].Exactly.Tolerations = []resourceapi.DeviceToleration{{
|
|
Effect: resourceapi.DeviceTaintEffectNoSchedule,
|
|
Operator: resourceapi.DeviceTolerationOpExists,
|
|
// No key: tolerate *all* taints with this effect.
|
|
}}
|
|
// Add a finalizer to ensure that we get a chance to test the pod status after eviction (= deletion).
|
|
pod.Finalizers = []string{"e2e-test/dont-delete-me"}
|
|
b.Create(tCtx, pod, template)
|
|
b.TestPod(tCtx, pod)
|
|
ginkgo.DeferCleanup(func(ctx context.Context) {
|
|
gomega.Eventually(ctx, func(ctx context.Context) error {
|
|
// Unblock shutdown by removing the finalizer.
|
|
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("get pod: %w", err)
|
|
}
|
|
pod.Finalizers = nil
|
|
_, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Update(ctx, pod, metav1.UpdateOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("remove finalizers from pod: %w", err)
|
|
}
|
|
return nil
|
|
}).WithTimeout(30*time.Second).WithPolling(1*time.Second).Should(gomega.Succeed(), "Failed to remove finalizers")
|
|
})
|
|
|
|
// Now evict it.
|
|
ginkgo.By("Evicting pod...")
|
|
taint := &resourcealphaapi.DeviceTaintRule{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "device-taint-rule-" + f.UniqueName + "-",
|
|
},
|
|
Spec: resourcealphaapi.DeviceTaintRuleSpec{
|
|
// All devices of the current driver instance.
|
|
DeviceSelector: &resourcealphaapi.DeviceTaintSelector{
|
|
Driver: &driver.Name,
|
|
},
|
|
Taint: resourcealphaapi.DeviceTaint{
|
|
Effect: resourcealphaapi.DeviceTaintEffectNoExecute,
|
|
Key: "test.example.com/evict",
|
|
Value: "now",
|
|
// No TimeAdded, gets defaulted.
|
|
},
|
|
},
|
|
}
|
|
createdTaint := b.Create(tCtx, taint)
|
|
taint = createdTaint[0].(*resourcealphaapi.DeviceTaintRule)
|
|
gomega.Expect(*taint).Should(gomega.HaveField("Spec.Taint.TimeAdded.Time", gomega.BeTemporally("~", time.Now(), time.Minute /* allow for some clock drift and delays */)))
|
|
framework.ExpectNoError(e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, pod.Name, f.Namespace.Name, f.Timeouts.PodStart))
|
|
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
framework.ExpectNoError(err, "get pod")
|
|
gomega.Expect(pod).Should(gomega.HaveField("Status.Conditions", gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
// LastTransitionTime is unknown.
|
|
"Type": gomega.Equal(v1.DisruptionTarget),
|
|
"Status": gomega.Equal(v1.ConditionTrue),
|
|
"Reason": gomega.Equal("DeletionByDeviceTaintManager"),
|
|
"Message": gomega.Equal("Device Taint manager: deleting due to NoExecute taint"),
|
|
}))))
|
|
})
|
|
})
|
|
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, f.WithFeatureGate(features.DRADeviceTaints), func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false), drautils.TaintAllDevices(resourceapi.DeviceTaint{
|
|
Key: "example.com/taint",
|
|
Value: "tainted",
|
|
Effect: resourceapi.DeviceTaintEffectNoExecute,
|
|
}))
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
f.It("NoExecute keeps pod pending", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
b.Create(tCtx, pod, template)
|
|
framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name))
|
|
})
|
|
|
|
f.It("NoExecute can be tolerated", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
template.Spec.Spec.Devices.Requests[0].Exactly.Tolerations = []resourceapi.DeviceToleration{{
|
|
Effect: resourceapi.DeviceTaintEffectNoExecute,
|
|
Operator: resourceapi.DeviceTolerationOpExists,
|
|
// No key: tolerate *all* taints with this effect.
|
|
}}
|
|
b.Create(tCtx, pod, template)
|
|
b.TestPod(tCtx, pod)
|
|
|
|
// Testing the pod is slow enough that the test fails when the pod gets evicted unexpectedly,
|
|
// We don't need to a "Consistently" here.
|
|
})
|
|
})
|
|
|
|
extendedResourceTest := func(ctx context.Context, b *drautils.Builder, f *framework.Framework, resourceNames []string, containerEnv []string) {
|
|
tCtx := f.TContext(ctx)
|
|
pod := b.Pod()
|
|
res := v1.ResourceList{}
|
|
res2 := v1.ResourceList{}
|
|
for _, resourceName := range resourceNames {
|
|
res[v1.ResourceName(resourceName)] = resource.MustParse("1")
|
|
res2[v1.ResourceName(resourceName)] = resource.MustParse("2")
|
|
}
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0], pod.Spec.Containers[0], pod.Spec.Containers[0]}
|
|
pod.Spec.InitContainers[0].Name += "-init"
|
|
// This must succeed for the pod to start.
|
|
pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env|grep container_1_request_0=true"}
|
|
pod.Spec.InitContainers[0].Resources.Requests = res2
|
|
pod.Spec.InitContainers[0].Resources.Limits = res2
|
|
pod.Spec.InitContainers[1].Name += "-sidecar"
|
|
// This must succeed for the pod to start.
|
|
pod.Spec.InitContainers[1].Command = []string{"sh", "-c", "while true; do env; env|grep container_1_request_0=true; echo $?; sleep 5; done"}
|
|
pod.Spec.InitContainers[1].RestartPolicy = ptr.To(v1.ContainerRestartPolicyAlways)
|
|
pod.Spec.InitContainers[1].Resources.Requests = res
|
|
pod.Spec.InitContainers[1].Resources.Limits = res
|
|
pod.Spec.InitContainers[2].Name += "-init-1"
|
|
// This must succeed for the pod to start.
|
|
pod.Spec.InitContainers[2].Command = []string{"sh", "-c", "env|grep container_3_request_0=true"}
|
|
pod.Spec.InitContainers[2].Resources.Requests = res
|
|
pod.Spec.InitContainers[2].Resources.Limits = res
|
|
|
|
b.Create(tCtx, pod)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
|
|
}
|
|
|
|
runExtendedClaimCleanup := func(ctx context.Context, b *drautils.Builder, shellScript string, waitFn func(context.Context, *v1.Pod) error, cleanupMessage string) {
|
|
tCtx := f.TContext(ctx)
|
|
pod := b.Pod()
|
|
res := v1.ResourceList{}
|
|
res[v1.ResourceName(b.ExtendedResourceName(0))] = resource.MustParse("1")
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
|
|
pod.Spec.Containers[0].Command = []string{"/bin/sh"}
|
|
pod.Spec.Containers[0].Args = []string{"-c", shellScript}
|
|
pod.Spec.RestartPolicy = v1.RestartPolicyNever
|
|
|
|
ginkgo.By("Creating the pod with extended resource")
|
|
b.Create(tCtx, pod)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
|
|
ginkgo.By("Verifying extended resource claim exists")
|
|
var extendedResourceClaim *resourceapi.ResourceClaim
|
|
gomega.Eventually(ctx, func(ctx context.Context) bool {
|
|
updatedPod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return false
|
|
}
|
|
if updatedPod.Status.ExtendedResourceClaimStatus == nil {
|
|
return false
|
|
}
|
|
claimName := updatedPod.Status.ExtendedResourceClaimStatus.ResourceClaimName
|
|
extendedResourceClaim, err = b.ClientV1(tCtx).ResourceClaims(pod.Namespace).Get(ctx, claimName, metav1.GetOptions{})
|
|
return err == nil && extendedResourceClaim != nil
|
|
}).WithTimeout(time.Minute).Should(gomega.BeTrueBecause("extended resource claim should be created"))
|
|
|
|
ginkgo.By("Waiting for the pod to reach terminal state")
|
|
err = waitFn(ctx, pod)
|
|
framework.ExpectNoError(err, "waiting for pod to reach terminal state")
|
|
|
|
ginkgo.By("Verifying extended resource claim is cleaned up")
|
|
gomega.Eventually(ctx, func(ctx context.Context) bool {
|
|
_, err := b.ClientV1(tCtx).ResourceClaims(pod.Namespace).Get(ctx, extendedResourceClaim.Name, metav1.GetOptions{})
|
|
return apierrors.IsNotFound(err)
|
|
}).WithTimeout(time.Minute).Should(gomega.BeTrueBecause("extended resource claim should be automatically deleted when pod %s", cleanupMessage))
|
|
}
|
|
|
|
framework.Context(f.WithFeatureGate(features.DRAExtendedResource), func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false))
|
|
b := drautils.NewBuilder(f, driver)
|
|
b.UseExtendedResourceName = true
|
|
|
|
ginkgo.It("must run a pod with extended resource with resource quota", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
hard := v1.ResourceList{
|
|
v1.ResourceName("count/resourceclaims.resource.k8s.io"): resource.MustParse("10"),
|
|
v1.ResourceName(fmt.Sprintf("requests.%s", b.ExtendedResourceName(0))): resource.MustParse("10"),
|
|
v1.ResourceName(fmt.Sprintf("requests.%s", "deviceclass.resource.kubernetes.io/"+b.ClassName())): resource.MustParse("10"),
|
|
v1.ResourceName(fmt.Sprintf("%s.deviceclass.resource.k8s.io/devices", b.Class(0).Name)): resource.MustParse("10"),
|
|
}
|
|
|
|
quota := &v1.ResourceQuota{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-quota", Namespace: f.Namespace.Name},
|
|
Spec: v1.ResourceQuotaSpec{
|
|
Hard: hard,
|
|
},
|
|
}
|
|
b.Create(tCtx, quota)
|
|
|
|
pod := b.Pod()
|
|
res := v1.ResourceList{}
|
|
|
|
// b.ExtendedResourceName(0) is added to the device class with name: b.ClassName()+"0"
|
|
res[v1.ResourceName(b.ExtendedResourceName(0))] = resource.MustParse("1")
|
|
// implicit extended resource name
|
|
res[v1.ResourceName("deviceclass.resource.kubernetes.io/"+b.ClassName())] = resource.MustParse("1")
|
|
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0], pod.Spec.Containers[0], pod.Spec.Containers[0]}
|
|
pod.Spec.InitContainers[0].Name += "-init"
|
|
// This must succeed for the pod to start.
|
|
pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env|grep request_0=true"}
|
|
pod.Spec.InitContainers[0].Resources.Requests = res
|
|
pod.Spec.InitContainers[0].Resources.Limits = res
|
|
pod.Spec.InitContainers[1].Name += "-sidecar"
|
|
// This must succeed for the pod to start.
|
|
pod.Spec.InitContainers[1].Command = []string{"sh", "-c", "env|grep container_1_request_0=true; sleep 1"}
|
|
pod.Spec.InitContainers[1].RestartPolicy = ptr.To(v1.ContainerRestartPolicyAlways)
|
|
pod.Spec.InitContainers[1].Resources.Requests = res
|
|
pod.Spec.InitContainers[1].Resources.Limits = res
|
|
pod.Spec.InitContainers[2].Name += "-init-1"
|
|
// This must succeed for the pod to start.
|
|
pod.Spec.InitContainers[2].Command = []string{"sh", "-c", "env|grep request_0=true"}
|
|
pod.Spec.InitContainers[2].Resources.Requests = res
|
|
pod.Spec.InitContainers[2].Resources.Limits = res
|
|
|
|
b.Create(tCtx, pod)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
containerEnv := []string{
|
|
"container_3_request_0", "true",
|
|
"container_3_request_1", "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
|
|
|
|
claim := b.ExternalClaim()
|
|
pod2 := b.PodExternal()
|
|
b.Create(tCtx, claim, pod2)
|
|
b.TestPod(tCtx, pod2)
|
|
|
|
// Expect 5 extended and implicit devices / requests resources to be consumed:
|
|
//
|
|
// - external claim, requesting 1 device:
|
|
// - 1 deviceclass
|
|
// - 1 extended request
|
|
// - 1 implicit deviceclaim request
|
|
//
|
|
// - pod, requesting 4 devices:
|
|
// - sidecar init container
|
|
// - 1 extended request
|
|
// - 1 implicit deviceclaim request
|
|
// - main container
|
|
// - 1 extended request
|
|
// - 1 implicit deviceclaim request
|
|
// - other init containers reuse the later sidecar/main container resource
|
|
//
|
|
// - extended resourceclaim auto-created from the pod, requesting 4 devices
|
|
// - 4 deviceclasses
|
|
// - 2 extended resource requests (4 minus 2 already counted from the pod)
|
|
// - 2 implicit deviceclaim requests (4 minus 2 already counted from the pod)
|
|
consumedDevices := "5"
|
|
|
|
usedResources := v1.ResourceList{}
|
|
usedResources[v1.ResourceName("count/resourceclaims.resource.k8s.io")] = resource.MustParse("2")
|
|
usedResources[v1.ResourceName(fmt.Sprintf("requests.%s", b.ExtendedResourceName(0)))] = resource.MustParse(consumedDevices)
|
|
usedResources[v1.ResourceName(fmt.Sprintf("requests.%s", "deviceclass.resource.kubernetes.io/"+b.ClassName()))] = resource.MustParse(consumedDevices)
|
|
usedResources[v1.ResourceName(fmt.Sprintf("%s.deviceclass.resource.k8s.io/devices", b.Class(0).Name))] = resource.MustParse(consumedDevices)
|
|
|
|
gomega.Eventually(ctx, framework.GetObject(f.ClientSet.CoreV1().ResourceQuotas(quota.Namespace).Get, quota.Name, metav1.GetOptions{})).
|
|
Should(gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Status": gomega.Equal(v1.ResourceQuotaStatus{
|
|
Hard: hard,
|
|
Used: usedResources,
|
|
})})))
|
|
|
|
framework.ExpectNoError(err)
|
|
})
|
|
|
|
ginkgo.It("must run a pod with both implicit and explicit extended resource with one container two resources", func(ctx context.Context) {
|
|
extendedResourceTest(ctx, b, f, []string{
|
|
// implicit extended resource name
|
|
"deviceclass.resource.kubernetes.io/" + b.ClassName(),
|
|
// b.ExtendedResourceName(0) is added to the device class with name: b.ClassName()+"0"
|
|
b.ExtendedResourceName(0),
|
|
}, []string{
|
|
"container_3_request_0", "true",
|
|
"container_3_request_1", "true",
|
|
})
|
|
})
|
|
|
|
ginkgo.It("must run a pod with extended resource with one container one resource", func(ctx context.Context) {
|
|
extendedResourceTest(ctx, b, f, []string{
|
|
b.ExtendedResourceName(0),
|
|
}, []string{
|
|
"container_3_request_0", "true",
|
|
})
|
|
})
|
|
|
|
ginkgo.It("must run a pod with extended resource with one container three resources", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objects []klog.KMetadata
|
|
for i := range 3 {
|
|
if i > 0 {
|
|
objects = append(objects, b.Class(i))
|
|
}
|
|
}
|
|
b.Create(tCtx, objects...)
|
|
extendedResourceTest(ctx, b, f, []string{
|
|
b.ExtendedResourceName(0),
|
|
b.ExtendedResourceName(1),
|
|
b.ExtendedResourceName(2),
|
|
}, []string{
|
|
"container_3_request_0", "true",
|
|
"container_3_request_1", "true",
|
|
"container_3_request_2", "true",
|
|
})
|
|
})
|
|
ginkgo.It("must run a pod with extended resource with three containers one resource each", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objects []klog.KMetadata
|
|
pod := b.Pod()
|
|
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
|
|
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
|
|
pod.Spec.Containers[0].Name = "container0"
|
|
pod.Spec.Containers[1].Name = "container1"
|
|
pod.Spec.Containers[2].Name = "container2"
|
|
objects = append(objects, pod)
|
|
|
|
for i := range 3 {
|
|
res := v1.ResourceList{}
|
|
res[v1.ResourceName(b.ExtendedResourceName(i))] = resource.MustParse("1")
|
|
pod.Spec.Containers[i].Resources.Requests = res
|
|
pod.Spec.Containers[i].Resources.Limits = res
|
|
if i > 0 {
|
|
objects = append(objects, b.Class(i))
|
|
}
|
|
}
|
|
|
|
b.Create(tCtx, objects...)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
for i := range 3 {
|
|
containerEnv := []string{
|
|
fmt.Sprintf("container_%d_request_0", i), "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[i].Name, false, containerEnv...)
|
|
}
|
|
})
|
|
ginkgo.It("must run a pod with extended resource with three containers multiple resources each", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objects []klog.KMetadata
|
|
pod := b.Pod()
|
|
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
|
|
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
|
|
pod.Spec.Containers[0].Name = "container0"
|
|
pod.Spec.Containers[1].Name = "container1"
|
|
pod.Spec.Containers[2].Name = "container2"
|
|
|
|
res := v1.ResourceList{}
|
|
res[v1.ResourceName(b.ExtendedResourceName(0))] = resource.MustParse("1")
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
res = v1.ResourceList{}
|
|
res[v1.ResourceName(b.ExtendedResourceName(1))] = resource.MustParse("1")
|
|
res[v1.ResourceName(b.ExtendedResourceName(2))] = resource.MustParse("1")
|
|
pod.Spec.Containers[1].Resources.Requests = res
|
|
pod.Spec.Containers[1].Resources.Limits = res
|
|
res = v1.ResourceList{}
|
|
res[v1.ResourceName(b.ExtendedResourceName(3))] = resource.MustParse("1")
|
|
res[v1.ResourceName(b.ExtendedResourceName(4))] = resource.MustParse("1")
|
|
res[v1.ResourceName(b.ExtendedResourceName(5))] = resource.MustParse("1")
|
|
pod.Spec.Containers[2].Resources.Requests = res
|
|
pod.Spec.Containers[2].Resources.Limits = res
|
|
for i := 1; i < 6; i++ {
|
|
objects = append(objects, b.Class(i))
|
|
}
|
|
objects = append(objects, pod)
|
|
|
|
b.Create(tCtx, objects...)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
containerEnv := []string{
|
|
"container_0_request_0", "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
|
|
containerEnv = []string{
|
|
"container_1_request_0", "true",
|
|
"container_1_request_1", "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[1].Name, false, containerEnv...)
|
|
containerEnv = []string{
|
|
"container_2_request_0", "true",
|
|
"container_2_request_1", "true",
|
|
"container_2_request_2", "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[2].Name, false, containerEnv...)
|
|
})
|
|
|
|
ginkgo.It("must cleanup extended resource claims when pods complete", func(ctx context.Context) {
|
|
runExtendedClaimCleanup(ctx, b,
|
|
"echo 'Pod started with extended resource'; sleep 2; echo 'Pod completed successfully'",
|
|
func(ctx context.Context, p *v1.Pod) error {
|
|
return e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, p.Name, p.Namespace)
|
|
},
|
|
"completes",
|
|
)
|
|
})
|
|
|
|
ginkgo.It("must cleanup extended resource claims when pods fail", func(ctx context.Context) {
|
|
runExtendedClaimCleanup(ctx, b,
|
|
"echo 'Pod started with extended resource'; sleep 2; echo 'Pod failing now'; exit 1",
|
|
func(ctx context.Context, p *v1.Pod) error {
|
|
return e2epod.WaitForPodCondition(ctx, f.ClientSet, p.Namespace, p.Name, "pod failed", framework.PodStartTimeout, func(pod *v1.Pod) (bool, error) {
|
|
return pod.Status.Phase == v1.PodFailed, nil
|
|
})
|
|
},
|
|
"fails",
|
|
)
|
|
})
|
|
f.It("process extended resources after device plugin uninstall", f.WithSerial(), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
resourceName := b.ExtendedResourceName(drautils.SingletonIndex)
|
|
extendedResourceName := deployDevicePlugin(ctx, f, nodes.NodeNames[0:1])
|
|
gomega.Expect(string(extendedResourceName)).To(gomega.Equal(resourceName))
|
|
|
|
getAllocatable := func() int {
|
|
node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodes.NodeNames[0], metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
for name, quantity := range node.Status.Allocatable {
|
|
if string(name) == resourceName {
|
|
return int(quantity.Value())
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
gomega.Eventually(ctx, getAllocatable).WithTimeout(f.Timeouts.PodStart).Should(gomega.Equal(2))
|
|
|
|
ginkgo.By("Uninstall Device Plugin")
|
|
err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).DeleteCollection(
|
|
ctx,
|
|
metav1.DeleteOptions{},
|
|
metav1.ListOptions{LabelSelector: "k8s-app=sample-device-plugin"},
|
|
)
|
|
framework.ExpectNoError(err, "uninstall device plugin")
|
|
|
|
ginkgo.By("Wait for NodeStatus.Allocatable = 0")
|
|
gomega.Eventually(ctx, getAllocatable).WithTimeout(f.Timeouts.PodDelete).Should(gomega.BeZero())
|
|
|
|
ginkgo.By("Create test pod")
|
|
pod := b.Pod()
|
|
pod.Spec.Containers[0].Name = "container0"
|
|
res := v1.ResourceList{
|
|
v1.ResourceName(resourceName): resource.MustParse("1"),
|
|
}
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
|
|
b.Create(tCtx, b.Class(drautils.SingletonIndex), pod)
|
|
|
|
err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
|
|
ginkgo.By("Check that pod is processed by the DRA driver")
|
|
containerEnv := []string{
|
|
"container_0_request_0", "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
|
|
})
|
|
|
|
ginkgo.It("must prevent overcommitment of extended resources", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
// Create first pod consuming all available resources
|
|
pod1 := b.Pod()
|
|
res := v1.ResourceList{
|
|
v1.ResourceName(b.ExtendedResourceName(0)): resource.MustParse("10"), // All available
|
|
}
|
|
pod1.Spec.Containers[0].Resources.Requests = res
|
|
pod1.Spec.Containers[0].Resources.Limits = res
|
|
b.Create(tCtx, pod1)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod1)
|
|
framework.ExpectNoError(err, "start pod")
|
|
|
|
ginkgo.By("Check that pod is processed by the DRA driver")
|
|
containerEnv := []string{
|
|
"container_0_request_0", "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, pod1, pod1.Spec.Containers[0].Name, false, containerEnv...)
|
|
|
|
// Second pod should remain unschedulable
|
|
pod2 := b.Pod()
|
|
pod2.Spec.Containers[0].Resources.Requests = res
|
|
pod2.Spec.Containers[0].Resources.Limits = res
|
|
b.Create(tCtx, pod2)
|
|
|
|
framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod2.Name, f.Namespace.Name))
|
|
|
|
// After deleting pod1, pod2 should be schedulable and run successfully
|
|
b.DeletePodAndWaitForNotFound(tCtx, pod1)
|
|
err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod2)
|
|
framework.ExpectNoError(err, "start pod")
|
|
drautils.TestContainerEnv(tCtx, pod2, pod2.Spec.Containers[0].Name, false, containerEnv...)
|
|
})
|
|
|
|
ginkgo.It("must reject pod with invalid extended resource name", func(ctx context.Context) {
|
|
pod := b.Pod()
|
|
res := v1.ResourceList{
|
|
v1.ResourceName("invalid_resource_name"): resource.MustParse("1"),
|
|
}
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
|
|
_, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
|
|
gomega.Expect(err).Should(gomega.HaveOccurred())
|
|
gomega.Expect(apierrors.IsInvalid(err)).Should(gomega.BeTrueBecause("pod with invalid extended resource name should be rejected"))
|
|
})
|
|
|
|
ginkgo.It("must accurately populate ExtendedResourceClaimStatus", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod := b.Pod()
|
|
resource0 := b.ExtendedResourceName(0)
|
|
resource1 := b.ExtendedResourceName(1)
|
|
res := v1.ResourceList{
|
|
v1.ResourceName(resource0): resource.MustParse("2"),
|
|
v1.ResourceName(resource1): resource.MustParse("1"),
|
|
}
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
|
|
b.Create(tCtx, b.Class(1), pod)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
|
|
ginkgo.By("Check that pod is processed by the DRA driver")
|
|
containerEnv := []string{
|
|
"container_0_request_0", "true",
|
|
"container_0_request_1", "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
|
|
|
|
// Verify status
|
|
updatedPod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
|
|
gomega.Expect(updatedPod.Status.ExtendedResourceClaimStatus).ShouldNot(gomega.BeNil())
|
|
gomega.Expect(updatedPod.Status.ExtendedResourceClaimStatus.ResourceClaimName).ShouldNot(gomega.BeEmpty())
|
|
|
|
// Verify mappings are correct
|
|
mappings := updatedPod.Status.ExtendedResourceClaimStatus.RequestMappings
|
|
gomega.Expect(mappings).Should(gomega.HaveLen(len(res)))
|
|
|
|
for i, mapping := range mappings {
|
|
gomega.Expect(mapping.ContainerName).Should(gomega.Equal(pod.Spec.Containers[0].Name))
|
|
gomega.Expect(v1.ResourceName(mapping.ResourceName)).Should(gomega.BeKeyOf(res))
|
|
gomega.Expect(mapping.RequestName).Should(gomega.Equal(fmt.Sprintf("container-0-request-%d", i)))
|
|
}
|
|
})
|
|
|
|
f.It("must run a pod with extended resource requesting zero", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
|
|
pod := b.Pod()
|
|
res := v1.ResourceList{
|
|
v1.ResourceName(b.ExtendedResourceName(0)): resource.MustParse("0"),
|
|
}
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
|
|
b.Create(tCtx, pod)
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
|
|
ginkgo.By("Check that pod is not processed by the DRA driver")
|
|
updatedPod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
gomega.Expect(updatedPod.Status.ExtendedResourceClaimStatus).Should(gomega.BeNil())
|
|
})
|
|
|
|
f.It("must process extended resource after Device Class creation", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
resourceName := b.ExtendedResourceName(1)
|
|
res := v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")}
|
|
pod := b.Pod()
|
|
pod.Spec.Containers[0].Resources.Requests = res
|
|
pod.Spec.Containers[0].Resources.Limits = res
|
|
|
|
b.Create(tCtx, pod)
|
|
|
|
err := e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)
|
|
framework.ExpectNoError(err, "pod should be unschedulable before device class creation")
|
|
|
|
ginkgo.By("Creating Device Class after pod creation")
|
|
b.Create(tCtx, b.Class(1))
|
|
|
|
err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
|
framework.ExpectNoError(err, "start pod")
|
|
|
|
ginkgo.By("Check that pod is processed by the DRA driver")
|
|
containerEnv := []string{"container_0_request_0", "true"}
|
|
drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
|
|
})
|
|
})
|
|
|
|
framework.Context(f.WithFeatureGate(features.DRAExtendedResource), func() {
|
|
nodes := drautils.NewNodes(f, 2, 2)
|
|
nodes.NumReservedNodes = 1
|
|
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(2, false))
|
|
b := drautils.NewBuilder(f, driver)
|
|
b.UseExtendedResourceName = true
|
|
|
|
// This test needs the entire test cluster for itself, one node in the cluster
|
|
// is deployed device plugin for the test, therefore it is marked as serial.
|
|
// The test runs two pods, one pod request extended resource backed by DRA,
|
|
// the other pod requests extended resource by device plugin.
|
|
f.It("must run pods with extended resource on dra nodes and device plugin nodes", f.WithSerial(), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
var objects []klog.KMetadata
|
|
extendedResourceName := deployDevicePlugin(ctx, f, nodes.ExtraNodeNames)
|
|
// b.ExtendedResourceName(SingletonIndex) must be the same as the returned extendedResourceName.
|
|
// b.ExtendedResourceName(SingletonIndex) is used for DRA drivers whereas
|
|
// extendedResourceName is used for device plugin.
|
|
gomega.Expect(string(extendedResourceName)).To(gomega.Equal(b.ExtendedResourceName(drautils.SingletonIndex)))
|
|
|
|
pod1 := b.Pod()
|
|
res := v1.ResourceList{}
|
|
res[v1.ResourceName(b.ExtendedResourceName(drautils.SingletonIndex))] = resource.MustParse("2")
|
|
pod1.Spec.Containers[0].Resources.Requests = res
|
|
pod1.Spec.Containers[0].Resources.Limits = res
|
|
objects = append(objects, b.Class(drautils.SingletonIndex), pod1)
|
|
|
|
pod2 := b.Pod()
|
|
pod2.Spec.Containers[0].Resources.Requests = res
|
|
pod2.Spec.Containers[0].Resources.Limits = res
|
|
objects = append(objects, pod2)
|
|
|
|
b.Create(tCtx, objects...)
|
|
|
|
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod1)
|
|
framework.ExpectNoError(err, "start pod1")
|
|
err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod2)
|
|
framework.ExpectNoError(err, "start pod2")
|
|
|
|
scheduledPod1, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod1.Name, metav1.GetOptions{})
|
|
gomega.Expect(scheduledPod1).ToNot(gomega.BeNil())
|
|
framework.ExpectNoError(err)
|
|
|
|
scheduledPod2, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod2.Name, metav1.GetOptions{})
|
|
gomega.Expect(scheduledPod2).ToNot(gomega.BeNil())
|
|
framework.ExpectNoError(err)
|
|
|
|
draPod := scheduledPod1
|
|
devicePluginPod := scheduledPod2
|
|
if scheduledPod1.Spec.NodeName == nodes.ExtraNodeNames[0] {
|
|
draPod = scheduledPod2
|
|
devicePluginPod = scheduledPod1
|
|
}
|
|
|
|
gomega.Expect(devicePluginPod.Spec.NodeName).To(gomega.Equal(nodes.ExtraNodeNames[0]))
|
|
gomega.Expect(devicePluginPod.Status.ExtendedResourceClaimStatus).To(gomega.BeNil())
|
|
gomega.Expect(draPod.Spec.NodeName).To(gomega.Equal(nodes.NodeNames[0]))
|
|
containerEnv := []string{
|
|
"container_0_request_0", "true",
|
|
}
|
|
drautils.TestContainerEnv(tCtx, draPod, draPod.Spec.Containers[0].Name, false, containerEnv...)
|
|
})
|
|
})
|
|
|
|
ginkgo.Context("ResourceSlice Controller", func() {
|
|
// This is a test for creating some large slices.
|
|
// Each slice is as large as API limits allow.
|
|
//
|
|
// Could become a conformance test because it only depends
|
|
// on the apiserver.
|
|
f.It("creates slices", func(ctx context.Context) {
|
|
dratest.TestCreateResourceSlices(f.TContext(ctx), 10)
|
|
})
|
|
})
|
|
|
|
framework.Context("control plane", func() {
|
|
nodes := drautils.NewNodes(f, 1, 1)
|
|
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false))
|
|
driver.WithKubelet = false
|
|
b := drautils.NewBuilder(f, driver)
|
|
|
|
f.It("validate ResourceClaimTemplate and ResourceClaim for admin access", f.WithFeatureGate(features.DRAAdminAccess), func(ctx context.Context) {
|
|
// Attempt to create claim and claim template with admin access. Must fail eventually.
|
|
claim := b.ExternalClaim()
|
|
claim.Spec.Devices.Requests[0].Exactly.AdminAccess = ptr.To(true)
|
|
_, claimTemplate := b.PodInline()
|
|
claimTemplate.Spec.Spec.Devices.Requests[0].Exactly.AdminAccess = ptr.To(true)
|
|
matchValidationError := gomega.MatchError(gomega.ContainSubstring("admin access to devices requires the `resource.kubernetes.io/admin-access: true` label on the containing namespace"))
|
|
gomega.Eventually(ctx, func(ctx context.Context) error {
|
|
// First delete, in case that it succeeded earlier.
|
|
if err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Delete(ctx, claim.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
|
return err
|
|
}
|
|
_, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Create(ctx, claim, metav1.CreateOptions{})
|
|
return err
|
|
}).Should(matchValidationError)
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) error {
|
|
// First delete, in case that it succeeded earlier.
|
|
if err := f.ClientSet.ResourceV1().ResourceClaimTemplates(f.Namespace.Name).Delete(ctx, claimTemplate.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
|
return err
|
|
}
|
|
_, err := f.ClientSet.ResourceV1().ResourceClaimTemplates(f.Namespace.Name).Create(ctx, claimTemplate, metav1.CreateOptions{})
|
|
return err
|
|
}).Should(matchValidationError)
|
|
|
|
// After labeling the namespace, creation must (eventually...) succeed.
|
|
_, err := f.ClientSet.CoreV1().Namespaces().Apply(ctx,
|
|
applyv1.Namespace(f.Namespace.Name).WithLabels(map[string]string{"resource.kubernetes.io/admin-access": "true"}),
|
|
metav1.ApplyOptions{FieldManager: f.UniqueName})
|
|
framework.ExpectNoError(err)
|
|
gomega.Eventually(ctx, func(ctx context.Context) error {
|
|
_, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Create(ctx, claim, metav1.CreateOptions{})
|
|
return err
|
|
}).Should(gomega.Succeed())
|
|
gomega.Eventually(ctx, func(ctx context.Context) error {
|
|
_, err := f.ClientSet.ResourceV1().ResourceClaimTemplates(f.Namespace.Name).Create(ctx, claimTemplate, metav1.CreateOptions{})
|
|
return err
|
|
}).Should(gomega.Succeed())
|
|
})
|
|
|
|
f.It("truncates the name of a generated resource claim", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod, template := b.PodInline()
|
|
pod.Name = strings.Repeat("p", 63)
|
|
pod.Spec.ResourceClaims[0].Name = strings.Repeat("c", 63)
|
|
pod.Spec.Containers[0].Resources.Claims[0].Name = pod.Spec.ResourceClaims[0].Name
|
|
b.Create(tCtx, template, pod)
|
|
|
|
b.TestPod(tCtx, pod)
|
|
})
|
|
|
|
f.It("supports count/resourceclaims.resource.k8s.io ResourceQuota", func(ctx context.Context) {
|
|
claim := &resourceapi.ResourceClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "claim-0",
|
|
Namespace: f.Namespace.Name,
|
|
},
|
|
Spec: resourceapi.ResourceClaimSpec{
|
|
Devices: resourceapi.DeviceClaim{
|
|
Requests: []resourceapi.DeviceRequest{{
|
|
Name: "req-0",
|
|
Exactly: &resourceapi.ExactDeviceRequest{
|
|
DeviceClassName: b.ClassName(),
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
}
|
|
_, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Create(ctx, claim, metav1.CreateOptions{})
|
|
framework.ExpectNoError(err, "create first claim")
|
|
|
|
resourceName := "count/resourceclaims.resource.k8s.io"
|
|
quota := &v1.ResourceQuota{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "object-count",
|
|
Namespace: f.Namespace.Name,
|
|
},
|
|
Spec: v1.ResourceQuotaSpec{
|
|
Hard: v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")},
|
|
},
|
|
}
|
|
quota, err = f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Create(ctx, quota, metav1.CreateOptions{})
|
|
framework.ExpectNoError(err, "create resource quota")
|
|
|
|
// Eventually the quota status should consider the existing claim.
|
|
gomega.Eventually(ctx, framework.GetObject(f.ClientSet.CoreV1().ResourceQuotas(quota.Namespace).Get, quota.Name, metav1.GetOptions{})).
|
|
Should(gstruct.PointTo(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Status": gomega.Equal(v1.ResourceQuotaStatus{
|
|
Hard: v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")},
|
|
Used: v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")},
|
|
})})))
|
|
|
|
// Now creating another claim should eventually fail. The quota may not be enforced immediately if
|
|
// it hasn't yet landed in the API server's cache.
|
|
//
|
|
// If creating a claim erroneously succeeds, we don't want to immediately fail on the next try
|
|
// with an "already exists" error, so use a new name each time.
|
|
claim.GenerateName = "claim-1-"
|
|
claim.Name = ""
|
|
gomega.Eventually(ctx, func(ctx context.Context) error {
|
|
_, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Create(ctx, claim, metav1.CreateOptions{})
|
|
return err
|
|
}).Should(gomega.MatchError(gomega.ContainSubstring("exceeded quota: object-count, requested: count/resourceclaims.resource.k8s.io=1, used: count/resourceclaims.resource.k8s.io=1, limited: count/resourceclaims.resource.k8s.io=1")), "creating second claim not allowed")
|
|
})
|
|
|
|
f.It("must be possible for the driver to update the ResourceClaim.Status.Devices once allocated", f.WithFeatureGate(features.DRAResourceClaimDeviceStatus), func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
pod := b.PodExternal()
|
|
claim := b.ExternalClaim()
|
|
b.Create(tCtx, claim, pod)
|
|
|
|
// Waits for the ResourceClaim to be allocated and the pod to be scheduled.
|
|
b.TestPod(tCtx, pod)
|
|
|
|
allocatedResourceClaim, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
gomega.Expect(allocatedResourceClaim).ToNot(gomega.BeNil())
|
|
gomega.Expect(allocatedResourceClaim.Status.Allocation).ToNot(gomega.BeNil())
|
|
gomega.Expect(allocatedResourceClaim.Status.Allocation.Devices.Results).To(gomega.HaveLen(1))
|
|
|
|
scheduledPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
gomega.Expect(scheduledPod).ToNot(gomega.BeNil())
|
|
|
|
shareID := (*string)(allocatedResourceClaim.Status.Allocation.Devices.Results[0].ShareID)
|
|
|
|
ginkgo.By("Setting the device status a first time")
|
|
allocatedResourceClaim.Status.Devices = append(allocatedResourceClaim.Status.Devices,
|
|
resourceapi.AllocatedDeviceStatus{
|
|
Driver: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Driver,
|
|
Pool: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Pool,
|
|
Device: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Device,
|
|
ShareID: shareID,
|
|
Conditions: []metav1.Condition{{Type: "a", Status: "True", Message: "c", Reason: "d", LastTransitionTime: metav1.NewTime(time.Now().Truncate(time.Second))}},
|
|
Data: &runtime.RawExtension{Raw: []byte(`{"foo":"bar"}`)},
|
|
NetworkData: &resourceapi.NetworkDeviceData{
|
|
InterfaceName: "inf1",
|
|
IPs: []string{"10.9.8.0/24", "2001:db8::/64"},
|
|
HardwareAddress: "bc:1c:b6:3e:b8:25",
|
|
},
|
|
})
|
|
|
|
// Updates the ResourceClaim from the driver on the same node as the pod.
|
|
plugin, ok := driver.Nodes[scheduledPod.Spec.NodeName]
|
|
if !ok {
|
|
framework.Failf("pod got scheduled to node %s without a plugin", scheduledPod.Spec.NodeName)
|
|
}
|
|
updatedResourceClaim, err := plugin.UpdateStatus(ctx, allocatedResourceClaim)
|
|
framework.ExpectNoError(err)
|
|
gomega.Expect(updatedResourceClaim).ToNot(gomega.BeNil())
|
|
gomega.Expect(updatedResourceClaim.Status.Devices).To(gomega.Equal(allocatedResourceClaim.Status.Devices))
|
|
|
|
ginkgo.By("Updating the device status")
|
|
updatedResourceClaim.Status.Devices[0] = resourceapi.AllocatedDeviceStatus{
|
|
Driver: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Driver,
|
|
Pool: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Pool,
|
|
Device: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Device,
|
|
ShareID: shareID,
|
|
Conditions: []metav1.Condition{{Type: "e", Status: "True", Message: "g", Reason: "h", LastTransitionTime: metav1.NewTime(time.Now().Truncate(time.Second))}},
|
|
Data: &runtime.RawExtension{Raw: []byte(`{"bar":"foo"}`)},
|
|
NetworkData: &resourceapi.NetworkDeviceData{
|
|
InterfaceName: "inf2",
|
|
IPs: []string{"10.9.8.1/24", "2001:db8::1/64"},
|
|
HardwareAddress: "bc:1c:b6:3e:b8:26",
|
|
},
|
|
}
|
|
|
|
updatedResourceClaim2, err := plugin.UpdateStatus(ctx, updatedResourceClaim)
|
|
framework.ExpectNoError(err)
|
|
gomega.Expect(updatedResourceClaim2).ToNot(gomega.BeNil())
|
|
gomega.Expect(updatedResourceClaim2.Status.Devices).To(gomega.Equal(updatedResourceClaim.Status.Devices))
|
|
|
|
getResourceClaim, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
gomega.Expect(getResourceClaim).ToNot(gomega.BeNil())
|
|
gomega.Expect(getResourceClaim.Status.Devices).To(gomega.Equal(updatedResourceClaim.Status.Devices))
|
|
})
|
|
})
|
|
|
|
framework.Context("control plane", func() {
|
|
nodes := drautils.NewNodes(f, 1, 4)
|
|
driver := drautils.NewDriver(f, nodes, drautils.DriverResources(1))
|
|
driver.WithKubelet = false
|
|
|
|
f.It("must apply per-node permission checks", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
// All of the operations use the client set of a kubelet plugin for
|
|
// a fictional node which both don't exist, so nothing interferes
|
|
// when we actually manage to create a slice.
|
|
fictionalNodeName := "dra-fictional-node"
|
|
gomega.Expect(nodes.NodeNames).NotTo(gomega.ContainElement(fictionalNodeName))
|
|
fictionalNodeClient := driver.ImpersonateKubeletPlugin(tCtx, &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: fictionalNodeName + "-dra-plugin",
|
|
Namespace: f.Namespace.Name,
|
|
UID: "12345",
|
|
},
|
|
Spec: v1.PodSpec{
|
|
NodeName: fictionalNodeName,
|
|
},
|
|
})
|
|
|
|
// This is for some actual node in the cluster.
|
|
realNodeName := nodes.NodeNames[0]
|
|
realNodeClient := driver.Nodes[realNodeName].ClientSet
|
|
|
|
// This is the slice that we try to create. It needs to be deleted
|
|
// after testing, if it still exists at that time.
|
|
fictionalNodeSlice := &resourceapi.ResourceSlice{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: fictionalNodeName + "-slice",
|
|
},
|
|
Spec: resourceapi.ResourceSliceSpec{
|
|
NodeName: ptr.To(fictionalNodeName),
|
|
Driver: "dra.example.com",
|
|
Pool: resourceapi.ResourcePool{
|
|
Name: "some-pool",
|
|
ResourceSliceCount: 1,
|
|
},
|
|
},
|
|
}
|
|
ginkgo.DeferCleanup(func(ctx context.Context) {
|
|
err := f.ClientSet.ResourceV1().ResourceSlices().Delete(ctx, fictionalNodeSlice.Name, metav1.DeleteOptions{})
|
|
if !apierrors.IsNotFound(err) {
|
|
framework.ExpectNoError(err)
|
|
}
|
|
})
|
|
|
|
// Messages from test-driver/deploy/example/plugin-permissions.yaml
|
|
matchVAPDeniedError := func(nodeName string, slice *resourceapi.ResourceSlice) types.GomegaMatcher {
|
|
subStr := fmt.Sprintf("this user running on node '%s' may not modify ", nodeName)
|
|
switch {
|
|
case ptr.Deref(slice.Spec.NodeName, "") != "":
|
|
subStr += fmt.Sprintf("resourceslices on node '%s'", *slice.Spec.NodeName)
|
|
default:
|
|
subStr += "cluster resourceslices"
|
|
}
|
|
return gomega.MatchError(gomega.ContainSubstring(subStr))
|
|
}
|
|
mustCreate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) *resourceapi.ResourceSlice {
|
|
ginkgo.GinkgoHelper()
|
|
slice, err := clientSet.ResourceV1().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{})
|
|
framework.ExpectNoError(err, fmt.Sprintf("CREATE: %s + %s", clientName, slice.Name))
|
|
return slice
|
|
}
|
|
mustUpdate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) *resourceapi.ResourceSlice {
|
|
ginkgo.GinkgoHelper()
|
|
slice, err := clientSet.ResourceV1().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{})
|
|
framework.ExpectNoError(err, fmt.Sprintf("UPDATE: %s + %s", clientName, slice.Name))
|
|
return slice
|
|
}
|
|
mustDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) {
|
|
ginkgo.GinkgoHelper()
|
|
err := clientSet.ResourceV1().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{})
|
|
framework.ExpectNoError(err, fmt.Sprintf("DELETE: %s + %s", clientName, slice.Name))
|
|
}
|
|
mustCreateAndDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice) {
|
|
ginkgo.GinkgoHelper()
|
|
slice = mustCreate(clientSet, clientName, slice)
|
|
mustDelete(clientSet, clientName, slice)
|
|
}
|
|
mustFailToCreate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) {
|
|
ginkgo.GinkgoHelper()
|
|
_, err := clientSet.ResourceV1().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{})
|
|
gomega.Expect(err).To(matchError, fmt.Sprintf("CREATE: %s + %s", clientName, slice.Name))
|
|
}
|
|
mustFailToUpdate := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) {
|
|
ginkgo.GinkgoHelper()
|
|
_, err := clientSet.ResourceV1().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{})
|
|
gomega.Expect(err).To(matchError, fmt.Sprintf("UPDATE: %s + %s", clientName, slice.Name))
|
|
}
|
|
mustFailToDelete := func(clientSet kubernetes.Interface, clientName string, slice *resourceapi.ResourceSlice, matchError types.GomegaMatcher) {
|
|
ginkgo.GinkgoHelper()
|
|
err := clientSet.ResourceV1().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{})
|
|
gomega.Expect(err).To(matchError, fmt.Sprintf("DELETE: %s + %s", clientName, slice.Name))
|
|
}
|
|
|
|
// Create with different clients, keep it in the end.
|
|
mustFailToCreate(realNodeClient, "real plugin", fictionalNodeSlice, matchVAPDeniedError(realNodeName, fictionalNodeSlice))
|
|
mustCreateAndDelete(fictionalNodeClient, "fictional plugin", fictionalNodeSlice)
|
|
createdFictionalNodeSlice := mustCreate(f.ClientSet, "admin", fictionalNodeSlice)
|
|
|
|
// Update with different clients.
|
|
mustFailToUpdate(realNodeClient, "real plugin", createdFictionalNodeSlice, matchVAPDeniedError(realNodeName, createdFictionalNodeSlice))
|
|
createdFictionalNodeSlice = mustUpdate(fictionalNodeClient, "fictional plugin", createdFictionalNodeSlice)
|
|
createdFictionalNodeSlice = mustUpdate(f.ClientSet, "admin", createdFictionalNodeSlice)
|
|
|
|
// Delete with different clients.
|
|
mustFailToDelete(realNodeClient, "real plugin", createdFictionalNodeSlice, matchVAPDeniedError(realNodeName, createdFictionalNodeSlice))
|
|
mustDelete(fictionalNodeClient, "fictional plugin", createdFictionalNodeSlice)
|
|
|
|
// Now the same for a slice which is not associated with a node.
|
|
clusterSlice := &resourceapi.ResourceSlice{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "cluster-slice",
|
|
},
|
|
Spec: resourceapi.ResourceSliceSpec{
|
|
AllNodes: ptr.To(true),
|
|
Driver: "another.example.com",
|
|
Pool: resourceapi.ResourcePool{
|
|
Name: "cluster-pool",
|
|
ResourceSliceCount: 1,
|
|
},
|
|
},
|
|
}
|
|
ginkgo.DeferCleanup(func(ctx context.Context) {
|
|
err := f.ClientSet.ResourceV1().ResourceSlices().Delete(ctx, clusterSlice.Name, metav1.DeleteOptions{})
|
|
if !apierrors.IsNotFound(err) {
|
|
framework.ExpectNoError(err)
|
|
}
|
|
})
|
|
|
|
// Create with different clients, keep it in the end.
|
|
mustFailToCreate(realNodeClient, "real plugin", clusterSlice, matchVAPDeniedError(realNodeName, clusterSlice))
|
|
mustFailToCreate(fictionalNodeClient, "fictional plugin", clusterSlice, matchVAPDeniedError(fictionalNodeName, clusterSlice))
|
|
createdClusterSlice := mustCreate(f.ClientSet, "admin", clusterSlice)
|
|
|
|
// Update with different clients.
|
|
mustFailToUpdate(realNodeClient, "real plugin", createdClusterSlice, matchVAPDeniedError(realNodeName, createdClusterSlice))
|
|
mustFailToUpdate(fictionalNodeClient, "fictional plugin", createdClusterSlice, matchVAPDeniedError(fictionalNodeName, createdClusterSlice))
|
|
createdClusterSlice = mustUpdate(f.ClientSet, "admin", createdClusterSlice)
|
|
|
|
// Delete with different clients.
|
|
mustFailToDelete(realNodeClient, "real plugin", createdClusterSlice, matchVAPDeniedError(realNodeName, createdClusterSlice))
|
|
mustFailToDelete(fictionalNodeClient, "fictional plugin", createdClusterSlice, matchVAPDeniedError(fictionalNodeName, createdClusterSlice))
|
|
mustDelete(f.ClientSet, "admin", createdClusterSlice)
|
|
})
|
|
})
|
|
|
|
multipleDrivers := func(nodeV1beta1, nodeV1 bool) {
|
|
nodes := drautils.NewNodes(f, 1, 4)
|
|
driver1 := drautils.NewDriver(f, nodes, drautils.DriverResources(2))
|
|
driver1.NodeV1beta1 = nodeV1beta1
|
|
driver1.NodeV1 = nodeV1
|
|
b1 := drautils.NewBuilder(f, driver1)
|
|
|
|
driver2 := drautils.NewDriver(f, nodes, drautils.DriverResources(2))
|
|
driver2.NodeV1beta1 = nodeV1beta1
|
|
driver2.NodeV1 = nodeV1
|
|
driver2.NameSuffix = "-other"
|
|
b2 := drautils.NewBuilder(f, driver2)
|
|
|
|
ginkgo.It("work", func(ctx context.Context) {
|
|
tCtx := f.TContext(ctx)
|
|
claim1 := b1.ExternalClaim()
|
|
claim1b := b1.ExternalClaim()
|
|
claim2 := b2.ExternalClaim()
|
|
claim2b := b2.ExternalClaim()
|
|
pod := b1.PodExternal()
|
|
for i, claim := range []*resourceapi.ResourceClaim{claim1b, claim2, claim2b} {
|
|
claim := claim
|
|
pod.Spec.ResourceClaims = append(pod.Spec.ResourceClaims,
|
|
v1.PodResourceClaim{
|
|
Name: fmt.Sprintf("claim%d", i+1),
|
|
ResourceClaimName: &claim.Name,
|
|
},
|
|
)
|
|
}
|
|
b1.Create(tCtx, claim1, claim1b, claim2, claim2b, pod)
|
|
b1.TestPod(tCtx, pod)
|
|
})
|
|
}
|
|
multipleDriversContext := func(prefix string, nodeV1beta1, nodeV1 bool) {
|
|
args := []any{
|
|
prefix,
|
|
func() {
|
|
multipleDrivers(nodeV1beta1, nodeV1)
|
|
},
|
|
}
|
|
if !nodeV1beta1 {
|
|
// If the v1beta1 gRPC API is disabled, then
|
|
// kubelet from 1.34 is required because that is
|
|
// when v1 was introduced.
|
|
args = append(args, f.WithLabel("KubeletMinVersion:1.34"))
|
|
}
|
|
framework.Context(args...)
|
|
}
|
|
|
|
framework.Context("kubelet", feature.DynamicResourceAllocation, "with multiple drivers", func() {
|
|
multipleDriversContext("using only drapbv1beta1", true, false)
|
|
multipleDriversContext("using only drapbv1", false, true)
|
|
multipleDriversContext("using drapbv1beta1 and drapbv1", true, true)
|
|
})
|
|
})
|