From 348f94ab55053ec9b4382de880406e27a730d8cd Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 11 Apr 2024 16:20:34 +0200 Subject: [PATCH] DRA: read ResourceClaim in DRA drivers This is the second and final step towards making kubelet independent of the resource.k8s.io API versioning because it now doesn't need to copy structs defined by that API from the driver to the API server. --- pkg/kubelet/cm/dra/manager.go | 20 +- pkg/kubelet/cm/dra/plugin/client_test.go | 7 +- .../kubelet/pkg/apis/dra/v1alpha3/api.pb.go | 202 +++--------------- .../kubelet/pkg/apis/dra/v1alpha3/api.proto | 10 - test/e2e/dra/test-driver/app/kubeletplugin.go | 110 +++++----- 5 files changed, 100 insertions(+), 249 deletions(-) diff --git a/pkg/kubelet/cm/dra/manager.go b/pkg/kubelet/cm/dra/manager.go index 18806006031..234b06eed0d 100644 --- a/pkg/kubelet/cm/dra/manager.go +++ b/pkg/kubelet/cm/dra/manager.go @@ -224,13 +224,9 @@ func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error { // Loop through all plugins and prepare for calling NodePrepareResources. for _, resourceHandle := range claimInfo.ResourceHandles { claim := &drapb.Claim{ - Namespace: claimInfo.Namespace, - Uid: string(claimInfo.ClaimUID), - Name: claimInfo.ClaimName, - ResourceHandle: resourceHandle.Data, - } - if resourceHandle.StructuredData != nil { - claim.StructuredResourceHandle = []*resourceapi.StructuredResourceHandle{resourceHandle.StructuredData} + Namespace: claimInfo.Namespace, + Uid: string(claimInfo.ClaimUID), + Name: claimInfo.ClaimName, } pluginName := resourceHandle.DriverName batches[pluginName] = append(batches[pluginName], claim) @@ -455,13 +451,9 @@ func (m *ManagerImpl) unprepareResources(podUID types.UID, namespace string, cla // Loop through all plugins and prepare for calling NodeUnprepareResources. for _, resourceHandle := range claimInfo.ResourceHandles { claim := &drapb.Claim{ - Namespace: claimInfo.Namespace, - Uid: string(claimInfo.ClaimUID), - Name: claimInfo.ClaimName, - ResourceHandle: resourceHandle.Data, - } - if resourceHandle.StructuredData != nil { - claim.StructuredResourceHandle = []*resourceapi.StructuredResourceHandle{resourceHandle.StructuredData} + Namespace: claimInfo.Namespace, + Uid: string(claimInfo.ClaimUID), + Name: claimInfo.ClaimName, } pluginName := resourceHandle.DriverName batches[pluginName] = append(batches[pluginName], claim) diff --git a/pkg/kubelet/cm/dra/plugin/client_test.go b/pkg/kubelet/cm/dra/plugin/client_test.go index b0e9ab058aa..5c6e46b33f0 100644 --- a/pkg/kubelet/cm/dra/plugin/client_test.go +++ b/pkg/kubelet/cm/dra/plugin/client_test.go @@ -135,10 +135,9 @@ func TestGRPCConnIsReused(t *testing.T) { req := &drapbv1alpha3.NodePrepareResourcesRequest{ Claims: []*drapbv1alpha3.Claim{ { - Namespace: "dummy-namespace", - Uid: "dummy-uid", - Name: "dummy-claim", - ResourceHandle: "dummy-resource", + Namespace: "dummy-namespace", + Uid: "dummy-uid", + Name: "dummy-claim", }, }, } diff --git a/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha3/api.pb.go b/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha3/api.pb.go index 3a9929690b1..c8186e8803a 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha3/api.pb.go +++ b/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha3/api.pb.go @@ -29,7 +29,6 @@ import ( codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" io "io" - v1alpha2 "k8s.io/api/resource/v1alpha2" math "math" math_bits "math/bits" reflect "reflect" @@ -351,18 +350,9 @@ type Claim struct { Uid string `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"` // The name of the Resource claim (ResourceClaim.meta.Name) // This field is REQUIRED. - Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` - // Resource handle (AllocationResult.ResourceHandles[*].Data) - // This field is REQUIRED. - ResourceHandle string `protobuf:"bytes,4,opt,name=resource_handle,json=resourceHandle,proto3" json:"resource_handle,omitempty"` - // Structured parameter resource handle (AllocationResult.ResourceHandles[*].StructuredData). - // This field is OPTIONAL. If present, it needs to be used - // instead of resource_handle. It will only have a single entry. - // - // Using "repeated" instead of "optional" is a workaround for https://github.com/gogo/protobuf/issues/713. - StructuredResourceHandle []*v1alpha2.StructuredResourceHandle `protobuf:"bytes,5,rep,name=structured_resource_handle,json=structuredResourceHandle,proto3" json:"structured_resource_handle,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Claim) Reset() { *m = Claim{} } @@ -418,20 +408,6 @@ func (m *Claim) GetName() string { return "" } -func (m *Claim) GetResourceHandle() string { - if m != nil { - return m.ResourceHandle - } - return "" -} - -func (m *Claim) GetStructuredResourceHandle() []*v1alpha2.StructuredResourceHandle { - if m != nil { - return m.StructuredResourceHandle - } - return nil -} - func init() { proto.RegisterType((*NodePrepareResourcesRequest)(nil), "v1alpha3.NodePrepareResourcesRequest") proto.RegisterType((*NodePrepareResourcesResponse)(nil), "v1alpha3.NodePrepareResourcesResponse") @@ -447,43 +423,37 @@ func init() { func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) } var fileDescriptor_00212fb1f9d3bf1c = []byte{ - // 562 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xce, 0x36, 0x49, 0x45, 0x26, 0x52, 0x8b, 0x56, 0x15, 0xb2, 0x42, 0x31, 0x91, 0x45, 0x49, - 0x0e, 0x60, 0x0b, 0x07, 0x50, 0x05, 0xe2, 0x92, 0x16, 0x54, 0x10, 0x42, 0xc8, 0x88, 0x0b, 0x97, - 0xb0, 0xb1, 0x07, 0xc7, 0x4a, 0x62, 0x9b, 0x5d, 0x3b, 0x52, 0x6f, 0x3c, 0x02, 0x8f, 0xd5, 0x03, - 0x07, 0xc4, 0x89, 0x53, 0x45, 0xcd, 0x8d, 0xa7, 0x40, 0x5e, 0xdb, 0x69, 0x13, 0x39, 0x4d, 0xa5, - 0xde, 0x66, 0xe7, 0xef, 0x9b, 0xfd, 0xe6, 0x07, 0x1a, 0x2c, 0xf4, 0xf4, 0x90, 0x07, 0x51, 0x40, - 0x6f, 0xcc, 0x1e, 0xb1, 0x49, 0x38, 0x62, 0xbd, 0xd6, 0x43, 0xd7, 0x8b, 0x46, 0xf1, 0x50, 0xb7, - 0x83, 0xa9, 0xe1, 0x06, 0x6e, 0x60, 0x48, 0x87, 0x61, 0xfc, 0x45, 0xbe, 0xe4, 0x43, 0x4a, 0x59, - 0x60, 0xeb, 0xc1, 0x78, 0x5f, 0xe8, 0x5e, 0x60, 0xb0, 0xd0, 0x33, 0x38, 0x8a, 0x20, 0xe6, 0x36, - 0x1a, 0x79, 0x32, 0xd3, 0x70, 0xd1, 0x47, 0xce, 0x22, 0x74, 0x32, 0x6f, 0xed, 0x15, 0xdc, 0x7e, - 0x17, 0x38, 0xf8, 0x9e, 0x63, 0xc8, 0x38, 0x5a, 0xb9, 0xbf, 0xb0, 0xf0, 0x6b, 0x8c, 0x22, 0xa2, - 0x1d, 0xd8, 0xb4, 0x27, 0xcc, 0x9b, 0x0a, 0x85, 0xb4, 0xab, 0xdd, 0xa6, 0xb9, 0xad, 0x17, 0x65, - 0xe9, 0x07, 0xa9, 0xde, 0xca, 0xcd, 0xda, 0x0f, 0x02, 0xbb, 0xe5, 0x89, 0x44, 0x18, 0xf8, 0x02, - 0xe9, 0x9b, 0xa5, 0x4c, 0xe6, 0x79, 0xa6, 0xcb, 0xe2, 0x32, 0x18, 0xf1, 0xd2, 0x8f, 0xf8, 0x71, - 0x01, 0xd6, 0xfa, 0x0c, 0xcd, 0x0b, 0x6a, 0x7a, 0x13, 0xaa, 0x63, 0x3c, 0x56, 0x48, 0x9b, 0x74, - 0x1b, 0x56, 0x2a, 0xd2, 0xe7, 0x50, 0x9f, 0xb1, 0x49, 0x8c, 0xca, 0x46, 0x9b, 0x74, 0x9b, 0xe6, - 0xde, 0xa5, 0x58, 0x05, 0x94, 0x95, 0xc5, 0x3c, 0xdb, 0xd8, 0x27, 0x9a, 0x53, 0x4a, 0xcb, 0xfc, - 0x33, 0x06, 0x34, 0x6d, 0xc7, 0x1b, 0x38, 0x38, 0xf3, 0x6c, 0xcc, 0x7e, 0xd4, 0xe8, 0x6f, 0x25, - 0xa7, 0x77, 0xe1, 0xe0, 0xf0, 0xf5, 0x61, 0xa6, 0xb5, 0xc0, 0x76, 0xbc, 0x5c, 0xa6, 0x3b, 0x50, - 0x47, 0xce, 0x03, 0x2e, 0x0b, 0x6a, 0x58, 0xd9, 0x43, 0x3b, 0x82, 0x3b, 0x29, 0xca, 0x47, 0x3f, - 0xbc, 0x2e, 0xfd, 0xbf, 0x08, 0xa8, 0xab, 0x52, 0xe5, 0x35, 0xbf, 0x5d, 0xca, 0xf5, 0x78, 0x91, - 0x94, 0xd5, 0x91, 0xa5, 0x2d, 0x18, 0xae, 0x6b, 0xc1, 0x8b, 0xc5, 0x16, 0x74, 0xd6, 0xa0, 0x95, - 0x35, 0xe1, 0xc9, 0x0a, 0x7a, 0xe6, 0x5f, 0x9a, 0xb3, 0x4a, 0x2e, 0xb2, 0xfa, 0x8f, 0x40, 0x5d, - 0xd6, 0x46, 0x77, 0xa1, 0xe1, 0xb3, 0x29, 0x8a, 0x90, 0xd9, 0x98, 0xfb, 0x9c, 0x2b, 0xd2, 0x9a, - 0x63, 0xcf, 0xc9, 0x3b, 0x92, 0x8a, 0x94, 0x42, 0x2d, 0x35, 0x2b, 0x55, 0xa9, 0x92, 0x32, 0xed, - 0xc0, 0x76, 0xb1, 0x45, 0x83, 0x11, 0xf3, 0x9d, 0x09, 0x2a, 0x35, 0x69, 0xde, 0x2a, 0xd4, 0x47, - 0x52, 0x4b, 0x23, 0x68, 0x89, 0x88, 0xc7, 0x76, 0x14, 0x73, 0x74, 0x06, 0xcb, 0x31, 0x75, 0xc9, - 0xf9, 0x53, 0x3d, 0x5b, 0x4e, 0x3d, 0xdd, 0xf3, 0xc2, 0xa5, 0x60, 0xc6, 0xd4, 0x3f, 0xcc, 0xe3, - 0xad, 0x85, 0xdc, 0x96, 0x22, 0x56, 0x58, 0xcc, 0x53, 0x02, 0xb5, 0x94, 0x24, 0xea, 0xc2, 0x4e, - 0xd9, 0x1e, 0xd1, 0xbd, 0x75, 0x7b, 0x26, 0x27, 0xad, 0x75, 0xff, 0x6a, 0xeb, 0xa8, 0x55, 0xe8, - 0x14, 0x6e, 0x95, 0xcf, 0x0b, 0xed, 0xac, 0x9f, 0xa8, 0x0c, 0xac, 0x7b, 0xd5, 0xd1, 0xd3, 0x2a, - 0xfd, 0xfe, 0xc9, 0x99, 0x4a, 0x7e, 0x9f, 0xa9, 0x95, 0x6f, 0x89, 0x4a, 0x4e, 0x12, 0x95, 0xfc, - 0x4c, 0x54, 0xf2, 0x27, 0x51, 0xc9, 0xf7, 0xbf, 0x6a, 0xe5, 0xd3, 0xbd, 0xfc, 0xd8, 0x8d, 0xe3, - 0x21, 0x4e, 0x30, 0x32, 0xc2, 0xb1, 0x9b, 0x1e, 0x3e, 0x61, 0x38, 0x9c, 0x15, 0x47, 0xaf, 0x37, - 0xdc, 0x94, 0xb7, 0xae, 0xf7, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xfd, 0x14, 0x30, 0xd4, 0x5f, 0x05, - 0x00, 0x00, + // 480 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xcd, 0x36, 0x4d, 0x85, 0x27, 0x12, 0xa0, 0x55, 0x85, 0xa2, 0x50, 0x4c, 0x64, 0x51, 0x92, + 0x0b, 0xb6, 0x48, 0x41, 0xaa, 0x40, 0x5c, 0xd2, 0x82, 0xf8, 0x12, 0x42, 0x96, 0xb8, 0x70, 0x81, + 0xb5, 0x3d, 0xb8, 0xab, 0x7c, 0xec, 0xb2, 0x6b, 0x47, 0xea, 0x8d, 0x9f, 0xc0, 0xcf, 0xea, 0x81, + 0x03, 0xe2, 0xc4, 0xa9, 0xa2, 0xe6, 0x8f, 0x20, 0xaf, 0x9d, 0xf4, 0x43, 0x4e, 0x5d, 0x89, 0xdb, + 0xcc, 0x78, 0x67, 0xde, 0x9b, 0xf7, 0x46, 0x06, 0x8b, 0x49, 0xee, 0x4a, 0x25, 0x12, 0x41, 0xaf, + 0xcd, 0x1f, 0xb2, 0x89, 0x3c, 0x60, 0x3b, 0xdd, 0x07, 0x31, 0x4f, 0x0e, 0xd2, 0xc0, 0x0d, 0xc5, + 0xd4, 0x8b, 0x45, 0x2c, 0x3c, 0xf3, 0x20, 0x48, 0xbf, 0x98, 0xcc, 0x24, 0x26, 0x2a, 0x1a, 0x9d, + 0x17, 0x70, 0xfb, 0x9d, 0x88, 0xf0, 0xbd, 0x42, 0xc9, 0x14, 0xfa, 0xa8, 0x45, 0xaa, 0x42, 0xd4, + 0x3e, 0x7e, 0x4d, 0x51, 0x27, 0xb4, 0x0f, 0x1b, 0xe1, 0x84, 0xf1, 0xa9, 0xee, 0x90, 0x5e, 0x73, + 0xd0, 0x1e, 0xde, 0x70, 0x17, 0x40, 0xee, 0x5e, 0x5e, 0xf7, 0xcb, 0xcf, 0xce, 0x0f, 0x02, 0x5b, + 0xd5, 0x83, 0xb4, 0x14, 0x33, 0x8d, 0xf4, 0xf5, 0x85, 0x49, 0xc3, 0xd3, 0x49, 0x97, 0xf5, 0x15, + 0x30, 0xfa, 0xf9, 0x2c, 0x51, 0x87, 0x0b, 0xb0, 0xee, 0x67, 0x68, 0x9f, 0x29, 0xd3, 0x9b, 0xd0, + 0x1c, 0xe3, 0x61, 0x87, 0xf4, 0xc8, 0xc0, 0xf2, 0xf3, 0x90, 0x3e, 0x85, 0xd6, 0x9c, 0x4d, 0x52, + 0xec, 0xac, 0xf5, 0xc8, 0xa0, 0x3d, 0xdc, 0xbe, 0x14, 0x6b, 0x01, 0xe5, 0x17, 0x3d, 0x4f, 0xd6, + 0x76, 0x89, 0x13, 0x55, 0xca, 0xb2, 0x5c, 0xc6, 0x83, 0x76, 0x18, 0xf1, 0x4f, 0x11, 0xce, 0x79, + 0x88, 0xc5, 0x46, 0xd6, 0xe8, 0x7a, 0x76, 0x7c, 0x17, 0xf6, 0xf6, 0x5f, 0xed, 0x17, 0x55, 0x1f, + 0xc2, 0x88, 0x97, 0x31, 0xdd, 0x84, 0x16, 0x2a, 0x25, 0x94, 0x21, 0x64, 0xf9, 0x45, 0xe2, 0xbc, + 0x84, 0x3b, 0x39, 0xca, 0x87, 0x99, 0xfc, 0x5f, 0xf9, 0x7f, 0x11, 0xb0, 0x57, 0x8d, 0x2a, 0x39, + 0xbf, 0xbd, 0x30, 0xeb, 0xd1, 0x79, 0x51, 0x56, 0x77, 0x56, 0x5a, 0x10, 0xd4, 0x59, 0xf0, 0xec, + 0xbc, 0x05, 0xfd, 0x1a, 0xb4, 0x2a, 0x13, 0x1e, 0xaf, 0x90, 0x67, 0xb9, 0xd2, 0x52, 0x55, 0x72, + 0x56, 0xd5, 0x37, 0xd0, 0x32, 0xd4, 0xe8, 0x16, 0x58, 0x33, 0x36, 0x45, 0x2d, 0x59, 0x88, 0xe5, + 0x93, 0xd3, 0x42, 0x4e, 0x39, 0xe5, 0x51, 0x69, 0x48, 0x1e, 0x52, 0x0a, 0xeb, 0xf9, 0xe7, 0x4e, + 0xd3, 0x94, 0x4c, 0x3c, 0x3c, 0x26, 0xb0, 0x9e, 0x93, 0xa0, 0x31, 0x6c, 0x56, 0xdd, 0x29, 0xdd, + 0xae, 0xbb, 0x63, 0xe3, 0x64, 0xf7, 0xfe, 0xd5, 0xce, 0xdd, 0x69, 0xd0, 0x29, 0xdc, 0xaa, 0xf6, + 0x83, 0xf6, 0xeb, 0x1d, 0x2b, 0xc0, 0x06, 0x57, 0xb5, 0xd6, 0x69, 0x8c, 0x46, 0x47, 0x27, 0x36, + 0xf9, 0x7d, 0x62, 0x37, 0xbe, 0x65, 0x36, 0x39, 0xca, 0x6c, 0xf2, 0x33, 0xb3, 0xc9, 0x9f, 0xcc, + 0x26, 0xdf, 0xff, 0xda, 0x8d, 0x8f, 0xf7, 0xc6, 0xbb, 0xda, 0xe5, 0xc2, 0x1b, 0xa7, 0x01, 0x4e, + 0x30, 0xf1, 0xe4, 0x38, 0xf6, 0x98, 0xe4, 0xda, 0x8b, 0x14, 0xf3, 0x16, 0x20, 0xc1, 0x86, 0xf9, + 0x97, 0xec, 0xfc, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xac, 0xa8, 0xa3, 0x6a, 0x91, 0x04, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -875,27 +845,6 @@ func (m *Claim) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.StructuredResourceHandle) > 0 { - for iNdEx := len(m.StructuredResourceHandle) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.StructuredResourceHandle[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintApi(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x2a - } - } - if len(m.ResourceHandle) > 0 { - i -= len(m.ResourceHandle) - copy(dAtA[i:], m.ResourceHandle) - i = encodeVarintApi(dAtA, i, uint64(len(m.ResourceHandle))) - i-- - dAtA[i] = 0x22 - } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) @@ -1055,16 +1004,6 @@ func (m *Claim) Size() (n int) { if l > 0 { n += 1 + l + sovApi(uint64(l)) } - l = len(m.ResourceHandle) - if l > 0 { - n += 1 + l + sovApi(uint64(l)) - } - if len(m.StructuredResourceHandle) > 0 { - for _, e := range m.StructuredResourceHandle { - l = e.Size() - n += 1 + l + sovApi(uint64(l)) - } - } return n } @@ -1169,17 +1108,10 @@ func (this *Claim) String() string { if this == nil { return "nil" } - repeatedStringForStructuredResourceHandle := "[]*StructuredResourceHandle{" - for _, f := range this.StructuredResourceHandle { - repeatedStringForStructuredResourceHandle += strings.Replace(fmt.Sprintf("%v", f), "StructuredResourceHandle", "v1alpha2.StructuredResourceHandle", 1) + "," - } - repeatedStringForStructuredResourceHandle += "}" s := strings.Join([]string{`&Claim{`, `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, `Uid:` + fmt.Sprintf("%v", this.Uid) + `,`, `Name:` + fmt.Sprintf("%v", this.Name) + `,`, - `ResourceHandle:` + fmt.Sprintf("%v", this.ResourceHandle) + `,`, - `StructuredResourceHandle:` + repeatedStringForStructuredResourceHandle + `,`, `}`, }, "") return s @@ -2039,72 +1971,6 @@ func (m *Claim) Unmarshal(dAtA []byte) error { } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ResourceHandle", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowApi - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthApi - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthApi - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ResourceHandle = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field StructuredResourceHandle", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowApi - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthApi - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthApi - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.StructuredResourceHandle = append(m.StructuredResourceHandle, &v1alpha2.StructuredResourceHandle{}) - if err := m.StructuredResourceHandle[len(m.StructuredResourceHandle)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipApi(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha3/api.proto b/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha3/api.proto index a4879e6c28a..46af3e07bc5 100644 --- a/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha3/api.proto +++ b/staging/src/k8s.io/kubelet/pkg/apis/dra/v1alpha3/api.proto @@ -22,7 +22,6 @@ package v1alpha3; option go_package = "k8s.io/kubelet/pkg/apis/dra/v1alpha3"; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; -import "k8s.io/api/resource/v1alpha2/generated.proto"; option (gogoproto.goproto_stringer_all) = false; option (gogoproto.stringer_all) = true; @@ -102,13 +101,4 @@ message Claim { // The name of the Resource claim (ResourceClaim.meta.Name) // This field is REQUIRED. string name = 3; - // Resource handle (AllocationResult.ResourceHandles[*].Data) - // This field is REQUIRED. - string resource_handle = 4; - // Structured parameter resource handle (AllocationResult.ResourceHandles[*].StructuredData). - // This field is OPTIONAL. If present, it needs to be used - // instead of resource_handle. It will only have a single entry. - // - // Using "repeated" instead of "optional" is a workaround for https://github.com/gogo/protobuf/issues/713. - repeated k8s.io.api.resource.v1alpha2.StructuredResourceHandle structured_resource_handle = 5; } diff --git a/test/e2e/dra/test-driver/app/kubeletplugin.go b/test/e2e/dra/test-driver/app/kubeletplugin.go index 00b56f3056d..b07c7a2c4a0 100644 --- a/test/e2e/dra/test-driver/app/kubeletplugin.go +++ b/test/e2e/dra/test-driver/app/kubeletplugin.go @@ -26,11 +26,12 @@ import ( "strings" "sync" - "github.com/google/go-cmp/cmp" "google.golang.org/grpc" resourceapi "k8s.io/api/resource/v1alpha2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/kubeletplugin" @@ -39,10 +40,11 @@ import ( ) type ExamplePlugin struct { - stopCh <-chan struct{} - logger klog.Logger - d kubeletplugin.DRAPlugin - fileOps FileOperations + stopCh <-chan struct{} + logger klog.Logger + kubeClient kubernetes.Interface + d kubeletplugin.DRAPlugin + fileOps FileOperations cdiDir string driverName string @@ -51,7 +53,7 @@ type ExamplePlugin struct { mutex sync.Mutex instancesInUse sets.Set[string] - prepared map[ClaimID]any + prepared map[ClaimID][]string // instance names gRPCCalls []GRPCCall blockPrepareResourcesMutex sync.Mutex @@ -129,13 +131,14 @@ func StartPlugin(ctx context.Context, cdiDir, driverName string, kubeClient kube ex := &ExamplePlugin{ stopCh: ctx.Done(), logger: logger, + kubeClient: kubeClient, fileOps: fileOps, cdiDir: cdiDir, driverName: driverName, nodeName: nodeName, instances: sets.New[string](), instancesInUse: sets.New[string](), - prepared: make(map[ClaimID]any), + prepared: make(map[ClaimID][]string), } for i := 0; i < ex.fileOps.NumResourceInstances; i++ { @@ -246,19 +249,47 @@ func (ex *ExamplePlugin) getUnprepareResourcesFailure() error { // a deterministic name to simplify NodeUnprepareResource (no need to remember // or discover the name) and idempotency (when called again, the file simply // gets written again). -func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName string, claimUID string, resourceHandle string, structuredResourceHandle []*resourceapi.StructuredResourceHandle) ([]string, error) { +func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimReq *drapbv1alpha3.Claim) ([]string, error) { logger := klog.FromContext(ctx) + // The plugin must retrieve the claim itself to get it in the version + // that it understands. + var resourceHandle string + var structuredResourceHandle *resourceapi.StructuredResourceHandle + claim, err := ex.kubeClient.ResourceV1alpha2().ResourceClaims(claimReq.Namespace).Get(ctx, claimReq.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("retrieve claim %s/%s: %w", claimReq.Namespace, claimReq.Name, err) + } + if claim.Status.Allocation == nil { + return nil, fmt.Errorf("claim %s/%s not allocated", claimReq.Namespace, claimReq.Name) + } + if claim.UID != types.UID(claimReq.Uid) { + return nil, fmt.Errorf("claim %s/%s got replaced", claimReq.Namespace, claimReq.Name) + } + haveResources := false + for _, handle := range claim.Status.Allocation.ResourceHandles { + if handle.DriverName == ex.driverName { + haveResources = true + resourceHandle = handle.Data + structuredResourceHandle = handle.StructuredData + break + } + } + if !haveResources { + // Nothing to do. + return nil, nil + } + ex.mutex.Lock() defer ex.mutex.Unlock() ex.blockPrepareResourcesMutex.Lock() defer ex.blockPrepareResourcesMutex.Unlock() - deviceName := "claim-" + claimUID + deviceName := "claim-" + claimReq.Uid vendor := ex.driverName class := "test" dev := vendor + "/" + class + "=" + deviceName - claimID := ClaimID{Name: claimName, UID: claimUID} + claimID := ClaimID{Name: claimReq.Name, UID: claimReq.Uid} if _, ok := ex.prepared[claimID]; ok { // Idempotent call, nothing to do. return []string{dev}, nil @@ -266,29 +297,22 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName stri // Determine environment variables. var p parameters - var actualResourceHandle any var instanceNames []string - switch len(structuredResourceHandle) { - case 0: + if structuredResourceHandle == nil { // Control plane controller did the allocation. if err := json.Unmarshal([]byte(resourceHandle), &p); err != nil { return nil, fmt.Errorf("unmarshal resource handle: %w", err) } - actualResourceHandle = resourceHandle - case 1: + } else { // Scheduler did the allocation with structured parameters. - handle := structuredResourceHandle[0] - if handle == nil { - return nil, errors.New("unexpected nil StructuredResourceHandle") - } - p.NodeName = handle.NodeName - if err := extractParameters(handle.VendorClassParameters, &p.EnvVars, "admin"); err != nil { + p.NodeName = structuredResourceHandle.NodeName + if err := extractParameters(structuredResourceHandle.VendorClassParameters, &p.EnvVars, "admin"); err != nil { return nil, err } - if err := extractParameters(handle.VendorClaimParameters, &p.EnvVars, "user"); err != nil { + if err := extractParameters(structuredResourceHandle.VendorClaimParameters, &p.EnvVars, "user"); err != nil { return nil, err } - for _, result := range handle.Results { + for _, result := range structuredResourceHandle.Results { if err := extractParameters(result.VendorRequestParameters, &p.EnvVars, "user"); err != nil { return nil, err } @@ -308,10 +332,6 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName stri } instanceNames = append(instanceNames, instanceName) } - actualResourceHandle = handle - default: - // Huh? - return nil, fmt.Errorf("invalid length of NodePrepareResourceRequest.StructuredResourceHandle: %d", len(structuredResourceHandle)) } // Sanity check scheduling. @@ -339,7 +359,7 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName stri }, }, } - filePath := ex.getJSONFilePath(claimUID) + filePath := ex.getJSONFilePath(claimReq.Uid) buffer, err := json.Marshal(spec) if err != nil { return nil, fmt.Errorf("marshal spec: %w", err) @@ -348,7 +368,7 @@ func (ex *ExamplePlugin) nodePrepareResource(ctx context.Context, claimName stri return nil, fmt.Errorf("failed to write CDI file %v", err) } - ex.prepared[claimID] = actualResourceHandle + ex.prepared[claimID] = instanceNames for _, instanceName := range instanceNames { ex.instancesInUse.Insert(instanceName) } @@ -384,7 +404,7 @@ func (ex *ExamplePlugin) NodePrepareResources(ctx context.Context, req *drapbv1a } for _, claimReq := range req.Claims { - cdiDevices, err := ex.nodePrepareResource(ctx, claimReq.Name, claimReq.Uid, claimReq.ResourceHandle, claimReq.StructuredResourceHandle) + cdiDevices, err := ex.nodePrepareResource(ctx, claimReq) if err != nil { resp.Claims[claimReq.Uid] = &drapbv1alpha3.NodePrepareResourceResponse{ Error: err.Error(), @@ -401,13 +421,13 @@ func (ex *ExamplePlugin) NodePrepareResources(ctx context.Context, req *drapbv1a // NodeUnprepareResource removes the CDI file created by // NodePrepareResource. It's idempotent, therefore it is not an error when that // file is already gone. -func (ex *ExamplePlugin) nodeUnprepareResource(ctx context.Context, claimName string, claimUID string, resourceHandle string, structuredResourceHandle []*resourceapi.StructuredResourceHandle) error { +func (ex *ExamplePlugin) nodeUnprepareResource(ctx context.Context, claimReq *drapbv1alpha3.Claim) error { ex.blockUnprepareResourcesMutex.Lock() defer ex.blockUnprepareResourcesMutex.Unlock() logger := klog.FromContext(ctx) - filePath := ex.getJSONFilePath(claimUID) + filePath := ex.getJSONFilePath(claimReq.Uid) if err := ex.fileOps.Remove(filePath); err != nil { return fmt.Errorf("error removing CDI file: %w", err) } @@ -416,33 +436,17 @@ func (ex *ExamplePlugin) nodeUnprepareResource(ctx context.Context, claimName st ex.mutex.Lock() defer ex.mutex.Unlock() - claimID := ClaimID{Name: claimName, UID: claimUID} - expectedResourceHandle, ok := ex.prepared[claimID] + claimID := ClaimID{Name: claimReq.Name, UID: claimReq.Uid} + instanceNames, ok := ex.prepared[claimID] if !ok { // Idempotent call, nothing to do. return nil } - var actualResourceHandle any = resourceHandle - if structuredResourceHandle != nil { - if len(structuredResourceHandle) != 1 { - return fmt.Errorf("unexpected number of entries in StructuredResourceHandle: %d", len(structuredResourceHandle)) - } - actualResourceHandle = structuredResourceHandle[0] - } - if diff := cmp.Diff(expectedResourceHandle, actualResourceHandle); diff != "" { - return fmt.Errorf("difference between expected (-) and actual resource handle (+):\n%s", diff) - } delete(ex.prepared, claimID) - if structuredResourceHandle := structuredResourceHandle; structuredResourceHandle != nil { - for _, handle := range structuredResourceHandle { - for _, result := range handle.Results { - instanceName := result.NamedResources.Name - ex.instancesInUse.Delete(instanceName) - } - } + for _, instanceName := range instanceNames { + ex.instancesInUse.Delete(instanceName) } - delete(ex.prepared, ClaimID{Name: claimName, UID: claimUID}) return nil } @@ -457,7 +461,7 @@ func (ex *ExamplePlugin) NodeUnprepareResources(ctx context.Context, req *drapbv } for _, claimReq := range req.Claims { - err := ex.nodeUnprepareResource(ctx, claimReq.Name, claimReq.Uid, claimReq.ResourceHandle, claimReq.StructuredResourceHandle) + err := ex.nodeUnprepareResource(ctx, claimReq) if err != nil { resp.Claims[claimReq.Uid] = &drapbv1alpha3.NodeUnprepareResourceResponse{ Error: err.Error(),