mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-18 13:57:38 +00:00
Merge pull request #136710 from bart0sh/PR221-integration-add-implicit-extended-resources
DRA: integration: test implicit resources
This commit is contained in:
Kubernetes Prow Robot
GitHub
@@ -77,37 +77,23 @@ import (
|
||||
var (
|
||||
// For more test data see pkg/scheduler/framework/plugin/dynamicresources/dynamicresources_test.go.
|
||||
|
||||
podName = "my-pod"
|
||||
podWithExtendedResourceName = "my-pod-with-extended-resource"
|
||||
namespace = "default"
|
||||
resourceName = "my-resource"
|
||||
extendedResourceName = "my-example.com/my-extended-resource"
|
||||
claimName = podName + "-" + resourceName
|
||||
className = "my-resource-class"
|
||||
extendedClassName = "my-extended-resource-class"
|
||||
device1 = "device-1"
|
||||
device2 = "device-2"
|
||||
podWithClaimName = st.MakePod().Name(podName).Namespace(namespace).
|
||||
Container("my-container").
|
||||
PodResourceClaims(v1.PodResourceClaim{Name: resourceName, ResourceClaimName: &claimName}).
|
||||
Obj()
|
||||
podWithExtendedResource = st.MakePod().Name(podWithExtendedResourceName).Namespace(namespace).
|
||||
podName = "my-pod"
|
||||
namespace = "default"
|
||||
resourceName = "my-resource"
|
||||
claimName = podName + "-" + resourceName
|
||||
className = "my-resource-class"
|
||||
device1 = "device-1"
|
||||
device2 = "device-2"
|
||||
|
||||
podWithClaimName = st.MakePod().Name(podName).Namespace(namespace).
|
||||
Container("my-container").
|
||||
Res(map[v1.ResourceName]string{v1.ResourceName(extendedResourceName): "1"}).
|
||||
PodResourceClaims(v1.PodResourceClaim{Name: resourceName, ResourceClaimName: &claimName}).
|
||||
Obj()
|
||||
class = &resourceapi.DeviceClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: className,
|
||||
},
|
||||
}
|
||||
classWithExtendedResource = &resourceapi.DeviceClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: extendedClassName,
|
||||
},
|
||||
Spec: resourceapi.DeviceClassSpec{
|
||||
ExtendedResourceName: &extendedResourceName,
|
||||
},
|
||||
}
|
||||
claim = st.MakeResourceClaim().
|
||||
Name(claimName).
|
||||
Namespace(namespace).
|
||||
@@ -177,7 +163,8 @@ func TestDRA(t *testing.T) {
|
||||
tCtx.Run("PublishResourceSlices", func(tCtx ktesting.TContext) {
|
||||
testPublishResourceSlices(tCtx, true, features.DRADeviceTaints, features.DRAPartitionableDevices, features.DRADeviceBindingConditions)
|
||||
})
|
||||
tCtx.Run("ExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, false) })
|
||||
tCtx.Run("ExplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, false, true) })
|
||||
tCtx.Run("ImplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, false, false) })
|
||||
tCtx.Run("ResourceClaimDeviceStatus", func(tCtx ktesting.TContext) { testResourceClaimDeviceStatus(tCtx, false) })
|
||||
tCtx.Run("DeviceBindingConditions", func(tCtx ktesting.TContext) { testDeviceBindingConditions(tCtx, false) })
|
||||
tCtx.Run("ResourceSliceController", func(tCtx ktesting.TContext) {
|
||||
@@ -251,8 +238,8 @@ func TestDRA(t *testing.T) {
|
||||
tCtx.Run("PrioritizedList", func(tCtx ktesting.TContext) { testPrioritizedList(tCtx, true) })
|
||||
tCtx.Run("PrioritizedListScoring", func(tCtx ktesting.TContext) { testPrioritizedListScoring(tCtx) })
|
||||
tCtx.Run("PublishResourceSlices", func(tCtx ktesting.TContext) { testPublishResourceSlices(tCtx, true) })
|
||||
// note testExtendedResource depends on testPublishResourceSlices to provide the devices
|
||||
tCtx.Run("ExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, true) })
|
||||
tCtx.Run("ExplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, true, true) })
|
||||
tCtx.Run("ImplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, true, false) })
|
||||
tCtx.Run("ResourceClaimDeviceStatus", func(tCtx ktesting.TContext) { testResourceClaimDeviceStatus(tCtx, true) })
|
||||
tCtx.Run("MaxResourceSlice", testMaxResourceSlice)
|
||||
tCtx.Run("EvictClusterWithRule", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, true) })
|
||||
@@ -800,55 +787,64 @@ func expectedAllocatedClaim(request string, nodeInfo nodeInfo) gtypes.GomegaMatc
|
||||
})))
|
||||
}
|
||||
|
||||
func testExtendedResource(tCtx ktesting.TContext, enabled bool) {
|
||||
func testExtendedResource(tCtx ktesting.TContext, enabled, explicit bool) {
|
||||
tCtx.Parallel()
|
||||
|
||||
namespace := createTestNamespace(tCtx, nil)
|
||||
driverName := namespace + driverNameSuffix
|
||||
class := classWithExtendedResource.DeepCopy()
|
||||
class.Spec.Selectors = []resourceapi.DeviceSelector{{
|
||||
CEL: &resourceapi.CELDeviceSelector{
|
||||
Expression: fmt.Sprintf("device.driver == %q", driverName),
|
||||
},
|
||||
}}
|
||||
c, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, class, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create class")
|
||||
|
||||
// Create a DeviceClass with or without ExtendedResourceName based on whether we're testing explicit or implicit extended resources.
|
||||
var resourceName string
|
||||
var spec *resourceapi.DeviceClassSpec
|
||||
if explicit {
|
||||
resourceName = "example.com/" + namespace
|
||||
// Set extended resource name in the DeviceClass spec only for explicit resources.
|
||||
// It's not required for implicit extended resources.
|
||||
spec = &resourceapi.DeviceClassSpec{
|
||||
ExtendedResourceName: &resourceName,
|
||||
}
|
||||
}
|
||||
class, driverName := createTestClassWithSpec(tCtx, namespace, spec)
|
||||
if explicit {
|
||||
if enabled {
|
||||
require.NotEmpty(tCtx, class.Spec.ExtendedResourceName, "should store ExtendedResourceName")
|
||||
} else {
|
||||
require.Empty(tCtx, class.Spec.ExtendedResourceName, "should strip ExtendedResourceName")
|
||||
}
|
||||
} else {
|
||||
// For implicit extended resources, derive the resource name from the class.
|
||||
resourceName = resourceapi.ResourceDeviceClassPrefix + class.Name
|
||||
}
|
||||
|
||||
slice := st.MakeResourceSlice("worker-0", driverName).Devices(device1)
|
||||
createSlice(tCtx, slice.Obj())
|
||||
|
||||
startScheduler(tCtx)
|
||||
|
||||
podWithOneContainer := st.MakePod().Name(podName).Namespace(namespace).Container("test-container").Obj()
|
||||
pod := createPodWithExtendedResource(tCtx, namespace, resourceName, "1", podWithOneContainer)
|
||||
|
||||
var schedulingAttempted gtypes.GomegaMatcher
|
||||
if enabled {
|
||||
require.NotEmpty(tCtx, c.Spec.ExtendedResourceName, "should store ExtendedResourceName")
|
||||
}
|
||||
|
||||
tCtx.Run("scheduler", func(tCtx ktesting.TContext) {
|
||||
startScheduler(tCtx)
|
||||
|
||||
pod := podWithExtendedResource.DeepCopy()
|
||||
pod.Namespace = namespace
|
||||
_, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create pod")
|
||||
schedulingAttempted := gomega.HaveField("Status.Conditions", gomega.ContainElement(
|
||||
// Scheduled using device1 in the slice above.
|
||||
schedulingAttempted = gomega.HaveField("Status.Conditions", gomega.ContainElement(
|
||||
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
||||
"Type": gomega.Equal(v1.PodScheduled),
|
||||
"Status": gomega.Equal(v1.ConditionTrue),
|
||||
}),
|
||||
))
|
||||
} else {
|
||||
schedulingAttempted = gomega.HaveField("Status.Conditions", gomega.ContainElement(
|
||||
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
||||
"Type": gomega.Equal(v1.PodScheduled),
|
||||
"Status": gomega.Equal(v1.ConditionFalse),
|
||||
"Reason": gomega.Equal("Unschedulable"),
|
||||
"Message": gomega.Equal("0/8 nodes are available: 8 Insufficient my-example.com/my-extended-resource. no new claims to deallocate, preemption: 0/8 nodes are available: 8 Preemption is not helpful for scheduling."),
|
||||
"Message": gomega.Equal(fmt.Sprintf("0/8 nodes are available: 8 Insufficient %s. no new claims to deallocate, preemption: 0/8 nodes are available: 8 Preemption is not helpful for scheduling.", resourceName)),
|
||||
}),
|
||||
))
|
||||
if enabled {
|
||||
// pod can be scheduled as the drivers in testPublishResourceSlices provide the devices.
|
||||
schedulingAttempted = gomega.HaveField("Status.Conditions", gomega.ContainElement(
|
||||
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
||||
"Type": gomega.Equal(v1.PodScheduled),
|
||||
"Status": gomega.Equal(v1.ConditionTrue),
|
||||
}),
|
||||
))
|
||||
}
|
||||
tCtx.Eventually(func(tCtx ktesting.TContext) (*v1.Pod, error) {
|
||||
return tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{})
|
||||
}).WithTimeout(time.Minute).WithPolling(time.Second).Should(schedulingAttempted)
|
||||
})
|
||||
}
|
||||
tCtx.Eventually(func(tCtx ktesting.TContext) (*v1.Pod, error) {
|
||||
return tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{})
|
||||
}).WithTimeout(time.Minute).WithPolling(time.Second).Should(schedulingAttempted)
|
||||
}
|
||||
|
||||
func testPublishResourceSlices(tCtx ktesting.TContext, haveLatestAPI bool, disabledFeatures ...featuregate.Feature) {
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourceapi "k8s.io/api/resource/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/kubernetes/test/utils/ktesting"
|
||||
"k8s.io/utils/ptr"
|
||||
@@ -88,6 +89,10 @@ func createSlice(tCtx ktesting.TContext, slice *resourceapi.ResourceSlice) *reso
|
||||
|
||||
// createTestClass creates a DeviceClass with a driver name derived from the test namespace
|
||||
func createTestClass(tCtx ktesting.TContext, namespace string) (*resourceapi.DeviceClass, string) {
|
||||
return createTestClassWithSpec(tCtx, namespace, nil)
|
||||
}
|
||||
|
||||
func createTestClassWithSpec(tCtx ktesting.TContext, namespace string, spec *resourceapi.DeviceClassSpec) (*resourceapi.DeviceClass, string) {
|
||||
tCtx.Helper()
|
||||
driverName := namespace + driverNameSuffix
|
||||
class := class.DeepCopy()
|
||||
@@ -97,7 +102,10 @@ func createTestClass(tCtx ktesting.TContext, namespace string) (*resourceapi.Dev
|
||||
Expression: fmt.Sprintf("device.driver == %q", driverName),
|
||||
},
|
||||
}}
|
||||
_, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, class, metav1.CreateOptions{})
|
||||
if spec != nil {
|
||||
class.Spec = *spec
|
||||
}
|
||||
class, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, class, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create class")
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.Log("Cleaning up DeviceClass...")
|
||||
@@ -138,6 +146,19 @@ func createClaim(tCtx ktesting.TContext, namespace string, suffix string, class
|
||||
|
||||
// createPod create a pod in the namespace, referencing the given claim.
|
||||
func createPod(tCtx ktesting.TContext, namespace string, suffix string, pod *v1.Pod, claims ...*resourceapi.ResourceClaim) *v1.Pod {
|
||||
return createPodInternal(tCtx, namespace, suffix, pod, nil, claims...)
|
||||
}
|
||||
|
||||
// createPodWithExtendedResource creates a pod in the namespace, requesting the given extended resource.
|
||||
func createPodWithExtendedResource(tCtx ktesting.TContext, namespace, resourceName, resourceQuantity string, pod *v1.Pod) *v1.Pod {
|
||||
return createPodInternal(tCtx, namespace, "", pod, map[v1.ResourceName]string{
|
||||
v1.ResourceName(resourceName): resourceQuantity,
|
||||
})
|
||||
}
|
||||
|
||||
// createPodInternal contains the common logic for createPod and createPodWithExtendedResource.
|
||||
// It shouldn't be called directly from tests.
|
||||
var createPodInternal = func(tCtx ktesting.TContext, namespace string, suffix string, pod *v1.Pod, resources map[v1.ResourceName]string, claims ...*resourceapi.ResourceClaim) *v1.Pod {
|
||||
tCtx.Helper()
|
||||
pod = pod.DeepCopy()
|
||||
pod.Name += suffix
|
||||
@@ -150,6 +171,19 @@ func createPod(tCtx ktesting.TContext, namespace string, suffix string, pod *v1.
|
||||
ResourceClaimName: &claim.Name,
|
||||
})
|
||||
}
|
||||
for res, qty := range resources {
|
||||
for i := range pod.Spec.Containers {
|
||||
container := &pod.Spec.Containers[i]
|
||||
if container.Resources.Requests == nil {
|
||||
container.Resources.Requests = v1.ResourceList{}
|
||||
}
|
||||
if container.Resources.Limits == nil {
|
||||
container.Resources.Limits = v1.ResourceList{}
|
||||
}
|
||||
container.Resources.Requests[res] = resource.MustParse(qty)
|
||||
container.Resources.Limits[res] = resource.MustParse(qty)
|
||||
}
|
||||
}
|
||||
pod.Spec.ResourceClaims = resourceClaims
|
||||
pod, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create pod "+podName)
|
||||
|
||||
Reference in New Issue
Block a user