diff --git a/test/integration/dra/dra_test.go b/test/integration/dra/dra_test.go index 502d2b7ea85..9cdc3ade3d8 100644 --- a/test/integration/dra/dra_test.go +++ b/test/integration/dra/dra_test.go @@ -77,37 +77,23 @@ import ( var ( // For more test data see pkg/scheduler/framework/plugin/dynamicresources/dynamicresources_test.go. - podName = "my-pod" - podWithExtendedResourceName = "my-pod-with-extended-resource" - namespace = "default" - resourceName = "my-resource" - extendedResourceName = "my-example.com/my-extended-resource" - claimName = podName + "-" + resourceName - className = "my-resource-class" - extendedClassName = "my-extended-resource-class" - device1 = "device-1" - device2 = "device-2" - podWithClaimName = st.MakePod().Name(podName).Namespace(namespace). - Container("my-container"). - PodResourceClaims(v1.PodResourceClaim{Name: resourceName, ResourceClaimName: &claimName}). - Obj() - podWithExtendedResource = st.MakePod().Name(podWithExtendedResourceName).Namespace(namespace). + podName = "my-pod" + namespace = "default" + resourceName = "my-resource" + claimName = podName + "-" + resourceName + className = "my-resource-class" + device1 = "device-1" + device2 = "device-2" + + podWithClaimName = st.MakePod().Name(podName).Namespace(namespace). Container("my-container"). - Res(map[v1.ResourceName]string{v1.ResourceName(extendedResourceName): "1"}). + PodResourceClaims(v1.PodResourceClaim{Name: resourceName, ResourceClaimName: &claimName}). Obj() class = &resourceapi.DeviceClass{ ObjectMeta: metav1.ObjectMeta{ Name: className, }, } - classWithExtendedResource = &resourceapi.DeviceClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: extendedClassName, - }, - Spec: resourceapi.DeviceClassSpec{ - ExtendedResourceName: &extendedResourceName, - }, - } claim = st.MakeResourceClaim(). Name(claimName). Namespace(namespace). @@ -177,7 +163,8 @@ func TestDRA(t *testing.T) { tCtx.Run("PublishResourceSlices", func(tCtx ktesting.TContext) { testPublishResourceSlices(tCtx, true, features.DRADeviceTaints, features.DRAPartitionableDevices, features.DRADeviceBindingConditions) }) - tCtx.Run("ExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, false) }) + tCtx.Run("ExplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, false, true) }) + tCtx.Run("ImplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, false, false) }) tCtx.Run("ResourceClaimDeviceStatus", func(tCtx ktesting.TContext) { testResourceClaimDeviceStatus(tCtx, false) }) tCtx.Run("DeviceBindingConditions", func(tCtx ktesting.TContext) { testDeviceBindingConditions(tCtx, false) }) tCtx.Run("ResourceSliceController", func(tCtx ktesting.TContext) { @@ -251,8 +238,8 @@ func TestDRA(t *testing.T) { tCtx.Run("PrioritizedList", func(tCtx ktesting.TContext) { testPrioritizedList(tCtx, true) }) tCtx.Run("PrioritizedListScoring", func(tCtx ktesting.TContext) { testPrioritizedListScoring(tCtx) }) tCtx.Run("PublishResourceSlices", func(tCtx ktesting.TContext) { testPublishResourceSlices(tCtx, true) }) - // note testExtendedResource depends on testPublishResourceSlices to provide the devices - tCtx.Run("ExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, true) }) + tCtx.Run("ExplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, true, true) }) + tCtx.Run("ImplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, true, false) }) tCtx.Run("ResourceClaimDeviceStatus", func(tCtx ktesting.TContext) { testResourceClaimDeviceStatus(tCtx, true) }) tCtx.Run("MaxResourceSlice", testMaxResourceSlice) tCtx.Run("EvictClusterWithRule", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, true) }) @@ -800,55 +787,64 @@ func expectedAllocatedClaim(request string, nodeInfo nodeInfo) gtypes.GomegaMatc }))) } -func testExtendedResource(tCtx ktesting.TContext, enabled bool) { +func testExtendedResource(tCtx ktesting.TContext, enabled, explicit bool) { tCtx.Parallel() namespace := createTestNamespace(tCtx, nil) - driverName := namespace + driverNameSuffix - class := classWithExtendedResource.DeepCopy() - class.Spec.Selectors = []resourceapi.DeviceSelector{{ - CEL: &resourceapi.CELDeviceSelector{ - Expression: fmt.Sprintf("device.driver == %q", driverName), - }, - }} - c, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, class, metav1.CreateOptions{FieldValidation: "Strict"}) - tCtx.ExpectNoError(err, "create class") + + // Create a DeviceClass with or without ExtendedResourceName based on whether we're testing explicit or implicit extended resources. + var resourceName string + var spec *resourceapi.DeviceClassSpec + if explicit { + resourceName = "example.com/" + namespace + // Set extended resource name in the DeviceClass spec only for explicit resources. + // It's not required for implicit extended resources. + spec = &resourceapi.DeviceClassSpec{ + ExtendedResourceName: &resourceName, + } + } + class, driverName := createTestClassWithSpec(tCtx, namespace, spec) + if explicit { + if enabled { + require.NotEmpty(tCtx, class.Spec.ExtendedResourceName, "should store ExtendedResourceName") + } else { + require.Empty(tCtx, class.Spec.ExtendedResourceName, "should strip ExtendedResourceName") + } + } else { + // For implicit extended resources, derive the resource name from the class. + resourceName = resourceapi.ResourceDeviceClassPrefix + class.Name + } slice := st.MakeResourceSlice("worker-0", driverName).Devices(device1) createSlice(tCtx, slice.Obj()) + startScheduler(tCtx) + + podWithOneContainer := st.MakePod().Name(podName).Namespace(namespace).Container("test-container").Obj() + pod := createPodWithExtendedResource(tCtx, namespace, resourceName, "1", podWithOneContainer) + + var schedulingAttempted gtypes.GomegaMatcher if enabled { - require.NotEmpty(tCtx, c.Spec.ExtendedResourceName, "should store ExtendedResourceName") - } - - tCtx.Run("scheduler", func(tCtx ktesting.TContext) { - startScheduler(tCtx) - - pod := podWithExtendedResource.DeepCopy() - pod.Namespace = namespace - _, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod, metav1.CreateOptions{FieldValidation: "Strict"}) - tCtx.ExpectNoError(err, "create pod") - schedulingAttempted := gomega.HaveField("Status.Conditions", gomega.ContainElement( + // Scheduled using device1 in the slice above. + schedulingAttempted = gomega.HaveField("Status.Conditions", gomega.ContainElement( + gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Type": gomega.Equal(v1.PodScheduled), + "Status": gomega.Equal(v1.ConditionTrue), + }), + )) + } else { + schedulingAttempted = gomega.HaveField("Status.Conditions", gomega.ContainElement( gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ "Type": gomega.Equal(v1.PodScheduled), "Status": gomega.Equal(v1.ConditionFalse), "Reason": gomega.Equal("Unschedulable"), - "Message": gomega.Equal("0/8 nodes are available: 8 Insufficient my-example.com/my-extended-resource. no new claims to deallocate, preemption: 0/8 nodes are available: 8 Preemption is not helpful for scheduling."), + "Message": gomega.Equal(fmt.Sprintf("0/8 nodes are available: 8 Insufficient %s. no new claims to deallocate, preemption: 0/8 nodes are available: 8 Preemption is not helpful for scheduling.", resourceName)), }), )) - if enabled { - // pod can be scheduled as the drivers in testPublishResourceSlices provide the devices. - schedulingAttempted = gomega.HaveField("Status.Conditions", gomega.ContainElement( - gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "Type": gomega.Equal(v1.PodScheduled), - "Status": gomega.Equal(v1.ConditionTrue), - }), - )) - } - tCtx.Eventually(func(tCtx ktesting.TContext) (*v1.Pod, error) { - return tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) - }).WithTimeout(time.Minute).WithPolling(time.Second).Should(schedulingAttempted) - }) + } + tCtx.Eventually(func(tCtx ktesting.TContext) (*v1.Pod, error) { + return tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, pod.Name, metav1.GetOptions{}) + }).WithTimeout(time.Minute).WithPolling(time.Second).Should(schedulingAttempted) } func testPublishResourceSlices(tCtx ktesting.TContext, haveLatestAPI bool, disabledFeatures ...featuregate.Feature) { diff --git a/test/integration/dra/helpers_test.go b/test/integration/dra/helpers_test.go index da1f5adefa0..aaabd362207 100644 --- a/test/integration/dra/helpers_test.go +++ b/test/integration/dra/helpers_test.go @@ -29,6 +29,7 @@ import ( v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/utils/ktesting" "k8s.io/utils/ptr" @@ -88,6 +89,10 @@ func createSlice(tCtx ktesting.TContext, slice *resourceapi.ResourceSlice) *reso // createTestClass creates a DeviceClass with a driver name derived from the test namespace func createTestClass(tCtx ktesting.TContext, namespace string) (*resourceapi.DeviceClass, string) { + return createTestClassWithSpec(tCtx, namespace, nil) +} + +func createTestClassWithSpec(tCtx ktesting.TContext, namespace string, spec *resourceapi.DeviceClassSpec) (*resourceapi.DeviceClass, string) { tCtx.Helper() driverName := namespace + driverNameSuffix class := class.DeepCopy() @@ -97,7 +102,10 @@ func createTestClass(tCtx ktesting.TContext, namespace string) (*resourceapi.Dev Expression: fmt.Sprintf("device.driver == %q", driverName), }, }} - _, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, class, metav1.CreateOptions{}) + if spec != nil { + class.Spec = *spec + } + class, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, class, metav1.CreateOptions{}) tCtx.ExpectNoError(err, "create class") tCtx.CleanupCtx(func(tCtx ktesting.TContext) { tCtx.Log("Cleaning up DeviceClass...") @@ -138,6 +146,19 @@ func createClaim(tCtx ktesting.TContext, namespace string, suffix string, class // createPod create a pod in the namespace, referencing the given claim. func createPod(tCtx ktesting.TContext, namespace string, suffix string, pod *v1.Pod, claims ...*resourceapi.ResourceClaim) *v1.Pod { + return createPodInternal(tCtx, namespace, suffix, pod, nil, claims...) +} + +// createPodWithExtendedResource creates a pod in the namespace, requesting the given extended resource. +func createPodWithExtendedResource(tCtx ktesting.TContext, namespace, resourceName, resourceQuantity string, pod *v1.Pod) *v1.Pod { + return createPodInternal(tCtx, namespace, "", pod, map[v1.ResourceName]string{ + v1.ResourceName(resourceName): resourceQuantity, + }) +} + +// createPodInternal contains the common logic for createPod and createPodWithExtendedResource. +// It shouldn't be called directly from tests. +var createPodInternal = func(tCtx ktesting.TContext, namespace string, suffix string, pod *v1.Pod, resources map[v1.ResourceName]string, claims ...*resourceapi.ResourceClaim) *v1.Pod { tCtx.Helper() pod = pod.DeepCopy() pod.Name += suffix @@ -150,6 +171,19 @@ func createPod(tCtx ktesting.TContext, namespace string, suffix string, pod *v1. ResourceClaimName: &claim.Name, }) } + for res, qty := range resources { + for i := range pod.Spec.Containers { + container := &pod.Spec.Containers[i] + if container.Resources.Requests == nil { + container.Resources.Requests = v1.ResourceList{} + } + if container.Resources.Limits == nil { + container.Resources.Limits = v1.ResourceList{} + } + container.Resources.Requests[res] = resource.MustParse(qty) + container.Resources.Limits[res] = resource.MustParse(qty) + } + } pod.Spec.ResourceClaims = resourceClaims pod, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod, metav1.CreateOptions{}) tCtx.ExpectNoError(err, "create pod "+podName)