Merge pull request #131983 from pohly/dra-resourceclaim-status-tests

DRA integration: move and extend device status test
This commit is contained in:
Kubernetes Prow Robot
2025-05-30 02:34:18 -07:00
committed by GitHub
3 changed files with 200 additions and 250 deletions

View File

@@ -17,7 +17,9 @@ limitations under the License.
package dra
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
@@ -41,8 +43,10 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilfeature "k8s.io/apiserver/pkg/util/feature"
resourceapiac "k8s.io/client-go/applyconfigurations/resource/v1beta1"
"k8s.io/client-go/informers"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -192,6 +196,28 @@ func TestDRA(t *testing.T) {
tCtx.Run("APIDisabled", testAPIDisabled)
},
},
"GA": {
// TODO (https://github.com/kubernetes/kubernetes/issues/131903): remove enabling the beta when promoting to GA.
apis: map[schema.GroupVersion]bool{
resourceapi.SchemeGroupVersion: true,
resourcev1beta2.SchemeGroupVersion: true,
},
features: map[featuregate.Feature]bool{
features.DynamicResourceAllocation: true,
// TODO: replace specific list with AllBeta once DRA is not beta.
features.DRAResourceClaimDeviceStatus: false,
// featuregate.Feature("AllBeta"): false,
},
f: func(tCtx ktesting.TContext) {
tCtx.Run("AdminAccess", func(tCtx ktesting.TContext) { testAdminAccess(tCtx, false) })
tCtx.Run("PrioritizedList", func(tCtx ktesting.TContext) { testPrioritizedList(tCtx, false) })
tCtx.Run("Pod", func(tCtx ktesting.TContext) { testPod(tCtx, true) })
tCtx.Run("PublishResourceSlices", func(tCtx ktesting.TContext) {
testPublishResourceSlices(tCtx, features.DRADeviceTaints, features.DRAPartitionableDevices)
})
tCtx.Run("ResourceClaimDeviceStatus", func(tCtx ktesting.TContext) { testResourceClaimDeviceStatus(tCtx, false) })
},
},
"core": {
apis: map[schema.GroupVersion]bool{
resourceapi.SchemeGroupVersion: true,
@@ -205,6 +231,7 @@ func TestDRA(t *testing.T) {
tCtx.Run("PublishResourceSlices", func(tCtx ktesting.TContext) {
testPublishResourceSlices(tCtx, features.DRADeviceTaints, features.DRAPartitionableDevices)
})
tCtx.Run("ResourceClaimDeviceStatus", func(tCtx ktesting.TContext) { testResourceClaimDeviceStatus(tCtx, true) })
},
},
"v1beta1": {
@@ -250,6 +277,7 @@ func TestDRA(t *testing.T) {
tCtx.Run("Convert", testConvert)
tCtx.Run("PrioritizedList", func(tCtx ktesting.TContext) { testPrioritizedList(tCtx, true) })
tCtx.Run("PublishResourceSlices", func(tCtx ktesting.TContext) { testPublishResourceSlices(tCtx) })
tCtx.Run("ResourceClaimDeviceStatus", func(tCtx ktesting.TContext) { testResourceClaimDeviceStatus(tCtx, true) })
tCtx.Run("MaxResourceSlice", testMaxResourceSlice)
},
},
@@ -668,6 +696,178 @@ func testPublishResourceSlices(tCtx ktesting.TContext, disabledFeatures ...featu
}
// testResourceClaimDeviceStatus creates a ResourceClaim with an invalid device (not allocated device)
// and checks that the object is not validated (feature enabled) resp. accepted without the field (disabled).
//
// When enabled, it tries server-side-apply (SSA) with different clients. This is what DRA drivers should be using.
func testResourceClaimDeviceStatus(tCtx ktesting.TContext, enabled bool) {
namespace := createTestNamespace(tCtx, nil)
claim := &resourceapi.ResourceClaim{
ObjectMeta: metav1.ObjectMeta{
Name: claimName,
},
Spec: resourceapi.ResourceClaimSpec{
Devices: resourceapi.DeviceClaim{
Requests: []resourceapi.DeviceRequest{
{
Name: "foo",
DeviceClassName: "foo",
},
},
},
},
}
claim, err := tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{})
tCtx.ExpectNoError(err, "create ResourceClaim")
deviceStatus := []resourceapi.AllocatedDeviceStatus{{
Driver: "one",
Pool: "global",
Device: "my-device",
Data: &runtime.RawExtension{
Raw: []byte(`{"kind": "foo", "apiVersion": "dra.example.com/v1"}`),
},
NetworkData: &resourceapi.NetworkDeviceData{
InterfaceName: "net-1",
IPs: []string{
"10.9.8.0/24",
"2001:db8::/64",
},
HardwareAddress: "ea:9f:cb:40:b1:7b",
},
}}
claim.Status.Devices = deviceStatus
updatedClaim, err := tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).UpdateStatus(tCtx, claim, metav1.UpdateOptions{})
if !enabled {
tCtx.ExpectNoError(err, "updating the status with an invalid AllocatedDeviceStatus should have worked because the field should have been dropped")
require.Empty(tCtx, updatedClaim.Status.Devices, "field should have been dropped")
return
}
// Tests for enabled feature follow.
if err == nil {
tCtx.Fatal("updating the status with an invalid AllocatedDeviceStatus should have failed and didn't")
}
// Add an allocation result.
claim.Status.Allocation = &resourceapi.AllocationResult{
Devices: resourceapi.DeviceAllocationResult{
Results: []resourceapi.DeviceRequestAllocationResult{
{
Request: "foo",
Driver: "one",
Pool: "global",
Device: "my-device",
},
{
Request: "foo",
Driver: "two",
Pool: "global",
Device: "another-device",
},
{
Request: "foo",
Driver: "three",
Pool: "global",
Device: "my-device",
},
},
},
}
claim, err = tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).UpdateStatus(tCtx, claim, metav1.UpdateOptions{})
tCtx.ExpectNoError(err, "add allocation result")
// Now adding the device status should work.
claim.Status.Devices = deviceStatus
claim, err = tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).UpdateStatus(tCtx, claim, metav1.UpdateOptions{})
tCtx.ExpectNoError(err, "add device status")
require.Equal(tCtx, deviceStatus, claim.Status.Devices, "after adding device status")
// Strip the RawExtension. SSA re-encodes it, which causes negligble differences that nonetheless break assert.Equal.
claim.Status.Devices[0].Data = nil
deviceStatus[0].Data = nil
claim, err = tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).UpdateStatus(tCtx, claim, metav1.UpdateOptions{})
tCtx.ExpectNoError(err, "add device status")
require.Equal(tCtx, deviceStatus, claim.Status.Devices, "after stripping RawExtension")
// Exercise SSA.
deviceStatusAC := resourceapiac.AllocatedDeviceStatus().
WithDriver("two").
WithPool("global").
WithDevice("another-device").
WithNetworkData(resourceapiac.NetworkDeviceData().WithInterfaceName("net-2"))
deviceStatus = append(deviceStatus, resourceapi.AllocatedDeviceStatus{
Driver: "two",
Pool: "global",
Device: "another-device",
NetworkData: &resourceapi.NetworkDeviceData{
InterfaceName: "net-2",
},
})
claimAC := resourceapiac.ResourceClaim(claim.Name, claim.Namespace).
WithStatus(resourceapiac.ResourceClaimStatus().WithDevices(deviceStatusAC))
claim, err = tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).ApplyStatus(tCtx, claimAC, metav1.ApplyOptions{
Force: true,
FieldManager: "manager-1",
})
tCtx.ExpectNoError(err, "apply device status two")
require.Equal(tCtx, deviceStatus, claim.Status.Devices, "after applying device status two")
deviceStatusAC = resourceapiac.AllocatedDeviceStatus().
WithDriver("three").
WithPool("global").
WithDevice("my-device").
WithNetworkData(resourceapiac.NetworkDeviceData().WithInterfaceName("net-3"))
deviceStatus = append(deviceStatus, resourceapi.AllocatedDeviceStatus{
Driver: "three",
Pool: "global",
Device: "my-device",
NetworkData: &resourceapi.NetworkDeviceData{
InterfaceName: "net-3",
},
})
claimAC = resourceapiac.ResourceClaim(claim.Name, claim.Namespace).
WithStatus(resourceapiac.ResourceClaimStatus().WithDevices(deviceStatusAC))
claim, err = tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).ApplyStatus(tCtx, claimAC, metav1.ApplyOptions{
Force: true,
FieldManager: "manager-2",
})
tCtx.ExpectNoError(err, "apply device status three")
require.Equal(tCtx, deviceStatus, claim.Status.Devices, "after applying device status three")
var buffer bytes.Buffer
encoder := json.NewEncoder(&buffer)
encoder.SetIndent(" ", " ")
tCtx.ExpectNoError(encoder.Encode(claim))
tCtx.Logf("Final ResourceClaim:\n%s", buffer.String())
// Update one entry, remove the other.
deviceStatusAC = resourceapiac.AllocatedDeviceStatus().
WithDriver("two").
WithPool("global").
WithDevice("another-device").
WithNetworkData(resourceapiac.NetworkDeviceData().WithInterfaceName("yet-another-net"))
deviceStatus[1].NetworkData.InterfaceName = "yet-another-net"
claimAC = resourceapiac.ResourceClaim(claim.Name, claim.Namespace).
WithStatus(resourceapiac.ResourceClaimStatus().WithDevices(deviceStatusAC))
claim, err = tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).ApplyStatus(tCtx, claimAC, metav1.ApplyOptions{
Force: true,
FieldManager: "manager-1",
})
tCtx.ExpectNoError(err, "update device status two")
require.Equal(tCtx, deviceStatus, claim.Status.Devices, "after updating device status two")
claimAC = resourceapiac.ResourceClaim(claim.Name, claim.Namespace)
deviceStatus = deviceStatus[0:2]
claim, err = tCtx.Client().ResourceV1beta1().ResourceClaims(namespace).ApplyStatus(tCtx, claimAC, metav1.ApplyOptions{
Force: true,
FieldManager: "manager-2",
})
tCtx.ExpectNoError(err, "remove device status three")
require.Equal(tCtx, deviceStatus, claim.Status.Devices, "after removing device status three")
}
// testMaxResourceSlice creates a ResourceSlice that is as large as possible
// and prints some information about it.
func testMaxResourceSlice(tCtx ktesting.TContext) {

View File

@@ -1,223 +0,0 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package resourceclaim
import (
"context"
"fmt"
"testing"
"k8s.io/api/resource/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/test/integration/framework"
)
// TestEnableDisableDRAResourceClaimDeviceStatus first test the feature gate disabled
// by creating a ResourceClaim with an invalid device (not allocated device) and checks
// the object is not validated.
// Then the feature gate is created, and an attempt to create similar invalid ResourceClaim
// is done with no success.
func TestEnableDisableDRAResourceClaimDeviceStatus(t *testing.T) {
// start etcd instance
etcdOptions := framework.SharedEtcd()
apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions()
// apiserver with the feature disabled
server1 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions,
[]string{
fmt.Sprintf("--runtime-config=%s=true", v1beta1.SchemeGroupVersion),
fmt.Sprintf("--feature-gates=%s=true,%s=false", features.DynamicResourceAllocation, features.DRAResourceClaimDeviceStatus),
},
etcdOptions)
client1, err := clientset.NewForConfig(server1.ClientConfig)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
ns := framework.CreateNamespaceOrDie(client1, "test-enable-dra-resourceclaim-device-status", t)
rcDisabledName := "test-enable-dra-resourceclaim-device-status-rc-disabled"
rcDisabled := &v1beta1.ResourceClaim{
ObjectMeta: metav1.ObjectMeta{
Name: rcDisabledName,
},
Spec: v1beta1.ResourceClaimSpec{
Devices: v1beta1.DeviceClaim{
Requests: []v1beta1.DeviceRequest{
{
Name: "foo",
DeviceClassName: "foo",
Count: 1,
AllocationMode: v1beta1.DeviceAllocationModeExactCount,
},
},
},
},
}
if _, err := client1.ResourceV1beta1().ResourceClaims(ns.Name).Create(context.TODO(), rcDisabled, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
rcDisabled.Status = v1beta1.ResourceClaimStatus{
Devices: []v1beta1.AllocatedDeviceStatus{
{
Driver: "foo",
Pool: "foo",
Device: "foo",
Data: &runtime.RawExtension{
Raw: []byte(`{"kind": "foo", "apiVersion": "dra.example.com/v1"}`),
},
NetworkData: &v1beta1.NetworkDeviceData{
InterfaceName: "net-1",
IPs: []string{
"10.9.8.0/24",
"2001:db8::/64",
},
HardwareAddress: "ea:9f:cb:40:b1:7b",
},
},
},
}
if _, err := client1.ResourceV1beta1().ResourceClaims(ns.Name).UpdateStatus(context.TODO(), rcDisabled, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
rcDisabled, err = client1.ResourceV1beta1().ResourceClaims(ns.Name).Get(context.TODO(), rcDisabledName, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
// No devices as the Kubernetes api-server dropped these fields since the feature is disabled.
if len(rcDisabled.Status.Devices) != 0 {
t.Fatalf("expected 0 Device in status got %d", len(rcDisabled.Status.Devices))
}
// shutdown apiserver with the feature disabled
server1.TearDownFn()
// apiserver with the feature enabled
server2 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions,
[]string{
fmt.Sprintf("--runtime-config=%s=true", v1beta1.SchemeGroupVersion),
fmt.Sprintf("--feature-gates=%s=true,%s=true", features.DynamicResourceAllocation, features.DRAResourceClaimDeviceStatus),
},
etcdOptions)
client2, err := clientset.NewForConfig(server2.ClientConfig)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
rcEnabledName := "test-enable-dra-resourceclaim-device-status-rc-enabled"
rcEnabled := &v1beta1.ResourceClaim{
ObjectMeta: metav1.ObjectMeta{
Name: rcEnabledName,
},
Spec: v1beta1.ResourceClaimSpec{
Devices: v1beta1.DeviceClaim{
Requests: []v1beta1.DeviceRequest{
{
Name: "bar",
DeviceClassName: "bar",
Count: 1,
AllocationMode: v1beta1.DeviceAllocationModeExactCount,
},
},
},
},
}
if _, err := client2.ResourceV1beta1().ResourceClaims(ns.Name).Create(context.TODO(), rcEnabled, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
// Tests the validation is enabled.
// validation will refuse this update as the device is not allocated.
rcEnabled.Status = v1beta1.ResourceClaimStatus{
Devices: []v1beta1.AllocatedDeviceStatus{
{
Driver: "bar",
Pool: "bar",
Device: "bar",
Data: &runtime.RawExtension{
Raw: []byte(`{"kind": "foo", "apiVersion": "dra.example.com/v1"}`),
},
NetworkData: &v1beta1.NetworkDeviceData{
InterfaceName: "net-1",
IPs: []string{
"10.9.8.0/24",
"2001:db8::/64",
},
HardwareAddress: "ea:9f:cb:40:b1:7b",
},
},
},
}
if _, err := client2.ResourceV1beta1().ResourceClaims(ns.Name).UpdateStatus(context.TODO(), rcEnabled, metav1.UpdateOptions{}); err == nil {
t.Fatalf("Expected error (must be an allocated device in the claim)")
}
rcEnabled.Status = v1beta1.ResourceClaimStatus{
Allocation: &v1beta1.AllocationResult{
Devices: v1beta1.DeviceAllocationResult{
Results: []v1beta1.DeviceRequestAllocationResult{
{
Request: "bar",
Driver: "bar",
Pool: "bar",
Device: "bar",
},
},
},
},
Devices: []v1beta1.AllocatedDeviceStatus{
{
Driver: "bar",
Pool: "bar",
Device: "bar",
Data: &runtime.RawExtension{
Raw: []byte(`{"kind": "foo", "apiVersion": "dra.example.com/v1"}`),
},
NetworkData: &v1beta1.NetworkDeviceData{
InterfaceName: "net-1",
IPs: []string{
"10.9.8.0/24",
"2001:db8::/64",
},
HardwareAddress: "ea:9f:cb:40:b1:7b",
},
},
},
}
if _, err := client2.ResourceV1beta1().ResourceClaims(ns.Name).UpdateStatus(context.TODO(), rcEnabled, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
// Tests the field is enabled.
rcEnabled, err = client2.ResourceV1beta1().ResourceClaims(ns.Name).Get(context.TODO(), rcEnabledName, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if len(rcEnabled.Status.Devices) != 1 {
t.Fatalf("expected 1 Device in status got %d", len(rcEnabled.Status.Devices))
}
// shutdown apiserver with the feature enabled
server2.TearDownFn()
}

View File

@@ -1,27 +0,0 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package resourceclaim
import (
"testing"
"k8s.io/kubernetes/test/integration/framework"
)
func TestMain(m *testing.M) {
framework.EtcdMain(m.Run)
}