Merge pull request #135037 from yliaog/extendedresourcecache

pick one device class deterministically for extended resource
This commit is contained in:
Kubernetes Prow Robot
2025-11-05 14:16:58 -08:00
committed by GitHub
8 changed files with 228 additions and 64 deletions

View File

@@ -145,6 +145,6 @@ func (s *sharedDRAManagerContract) DeviceClassResolver() fwk.DeviceClassResolver
type deviceClassResolverContract struct{}
func (d *deviceClassResolverContract) GetDeviceClass(_ v1.ResourceName) string {
return ""
func (d *deviceClassResolverContract) GetDeviceClass(_ v1.ResourceName) *resourceapi.DeviceClass {
return nil
}

View File

@@ -51,7 +51,6 @@ type DefaultDRAManager struct {
func NewDRAManager(ctx context.Context, claimsCache *assumecache.AssumeCache, resourceSliceTracker *resourceslicetracker.Tracker, informerFactory informers.SharedInformerFactory) *DefaultDRAManager {
logger := klog.FromContext(ctx)
manager := &DefaultDRAManager{
resourceClaimTracker: &claimTracker{
cache: claimsCache,
@@ -91,7 +90,7 @@ func (s *DefaultDRAManager) DeviceClasses() fwk.DeviceClassLister {
// disabled.
//
// That's okay, extendedresourcecache.ExtendedResourceCache.GetDeviceClass
// returns the empty string if called for nil.
// returns nil if called for nil.
func (s *DefaultDRAManager) DeviceClassResolver() fwk.DeviceClassResolver {
return s.extendedResourceCache
}

View File

@@ -418,7 +418,7 @@ func hasDeviceClassMappedExtendedResource(reqs v1.ResourceList, cache fwk.Device
continue
}
if schedutil.IsDRAExtendedResourceName(rName) {
if cache.GetDeviceClass(rName) != "" {
if cache.GetDeviceClass(rName) != nil {
return true
}
}
@@ -754,7 +754,7 @@ func (pl *DynamicResources) filterExtendedResources(state *stateData, pod *v1.Po
continue
}
allocatable, okScalar := nodeInfo.GetAllocatable().GetScalarResources()[rName]
isBackedByDRA := cache.GetDeviceClass(rName) != ""
isBackedByDRA := cache.GetDeviceClass(rName) != nil
if isBackedByDRA {
if allocatable > 0 {
// node provides the resource via device plugin
@@ -829,9 +829,9 @@ func createDeviceRequests(pod *v1.Pod, extendedResources map[v1.ResourceName]int
if !ok || crq == 0 {
continue
}
className := cache.GetDeviceClass(r)
class := cache.GetDeviceClass(r)
// skip if the request does not map to a device class
if className == "" {
if class == nil {
continue
}
keys := make([]string, 0, len(creqs))
@@ -855,7 +855,7 @@ func createDeviceRequests(pod *v1.Pod, extendedResources map[v1.ResourceName]int
resourceapi.DeviceRequest{
Name: fmt.Sprintf("container-%d-request-%d", i, ridx), // need to be container name index - extended resource name index
Exactly: &resourceapi.ExactDeviceRequest{
DeviceClassName: className, // map external resource name -> device class name
DeviceClassName: class.Name, // map external resource name -> device class name
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
Count: crq,
},

View File

@@ -3151,10 +3151,10 @@ func Test_isSchedulableAfterPodChange(t *testing.T) {
// mockDeviceClassResolver is a simple mock implementation of fwk.DeviceClassResolver for testing
type mockDeviceClassResolver struct {
mapping map[v1.ResourceName]string
mapping map[v1.ResourceName]*resourceapi.DeviceClass
}
func (m *mockDeviceClassResolver) GetDeviceClass(resourceName v1.ResourceName) string {
func (m *mockDeviceClassResolver) GetDeviceClass(resourceName v1.ResourceName) *resourceapi.DeviceClass {
return m.mapping[resourceName]
}
@@ -3197,16 +3197,36 @@ func Test_createDeviceRequests(t *testing.T) {
v1.ResourceName(extendedResourceName): 1,
v1.ResourceName(extendedResourceName + "init"): 2,
}
devMap := map[v1.ResourceName]string{
v1.ResourceName(extendedResourceName): "class",
devMap := map[v1.ResourceName]*resourceapi.DeviceClass{
v1.ResourceName(extendedResourceName): {
ObjectMeta: metav1.ObjectMeta{
Name: "class",
},
},
}
devMap2 := map[v1.ResourceName]string{
v1.ResourceName(extendedResourceName): "class",
v1.ResourceName(extendedResourceName + "1"): "class1",
devMap2 := map[v1.ResourceName]*resourceapi.DeviceClass{
v1.ResourceName(extendedResourceName): {
ObjectMeta: metav1.ObjectMeta{
Name: "class",
},
},
v1.ResourceName(extendedResourceName + "1"): {
ObjectMeta: metav1.ObjectMeta{
Name: "class1",
},
},
}
devMapInit := map[v1.ResourceName]string{
v1.ResourceName(extendedResourceName): "class",
v1.ResourceName(extendedResourceName + "init"): "classInit",
devMapInit := map[v1.ResourceName]*resourceapi.DeviceClass{
v1.ResourceName(extendedResourceName): {
ObjectMeta: metav1.ObjectMeta{
Name: "class",
},
},
v1.ResourceName(extendedResourceName + "init"): {
ObjectMeta: metav1.ObjectMeta{
Name: "classInit",
},
},
}
devReq := resourceapi.DeviceRequest{
Name: "container-0-request-0",

View File

@@ -210,7 +210,7 @@ func shouldDelegateResourceToDRA(rName v1.ResourceName, nodeInfo fwk.NodeInfo, d
// If draManager is available, check the cache for a mapping
if draManager != nil {
cache := draManager.DeviceClassResolver()
return cache.GetDeviceClass(rName) != ""
return cache.GetDeviceClass(rName) != nil
}
// If draManager is nil (e.g., kubelet admission check), delegate resources that are not in

View File

@@ -17,6 +17,7 @@ limitations under the License.
package extendedresourcecache
import (
"cmp"
"fmt"
"sync"
@@ -34,8 +35,8 @@ type ExtendedResourceCache struct {
handlers []cache.ResourceEventHandler
mutex sync.RWMutex
// mapping maps extended resource name to device class name
mapping map[v1.ResourceName]string
// mapping maps extended resource name to device class
mapping map[v1.ResourceName]*resourceapi.DeviceClass
}
var _ cache.ResourceEventHandler = &ExtendedResourceCache{}
@@ -48,7 +49,7 @@ func NewExtendedResourceCache(logger klog.Logger, handlers ...cache.ResourceEven
cache := &ExtendedResourceCache{
logger: logger,
handlers: handlers,
mapping: make(map[v1.ResourceName]string),
mapping: make(map[v1.ResourceName]*resourceapi.DeviceClass),
}
return cache
@@ -65,14 +66,14 @@ func (c *ExtendedResourceCache) AddEventHandler(handler cache.ResourceEventHandl
c.handlers = append(c.handlers, handler)
}
// GetDeviceClass returns the device class name for the given extended resource name.
// Returns empty string if the resource name is not found in the cache.
// GetDeviceClass returns the device class for the given extended resource name.
// Returns nil if the resource name is not found in the cache.
//
// This (and only this) method may be called on a nil ExtendedResourceCache. The nil
// instance always returns the empty string.
func (c *ExtendedResourceCache) GetDeviceClass(resourceName v1.ResourceName) string {
// instance always returns nil.
func (c *ExtendedResourceCache) GetDeviceClass(resourceName v1.ResourceName) *resourceapi.DeviceClass {
if c == nil {
return ""
return nil
}
c.mutex.RLock()
defer c.mutex.RUnlock()
@@ -139,6 +140,34 @@ func (c *ExtendedResourceCache) updateMapping(newDeviceClass, oldDeviceClass *re
c.mutex.Lock()
defer c.mutex.Unlock()
// Different DeviceClasses could specify the same ExtendedResourceName.
// If we find such a clash, we need to pick one of them.
//
// Such a clash should be rare, but can occur while migrating from one
// DeviceClass to another. To support such a migration, we prefer the
// newer DeviceClass. The name serves as tie-breaker.
//
// The implicit deviceclass.resource.kubernetes.io/<class name> is always
// unique and also cannot appear in a different class as ExtendedResourceName
// (prevented by validation), so we don't need to do the same check for
// implicit mappings.
var classWithSameExtendedResourceName *resourceapi.DeviceClass
if newDeviceClass.Spec.ExtendedResourceName != nil {
classWithSameExtendedResourceName = c.mapping[v1.ResourceName(*newDeviceClass.Spec.ExtendedResourceName)]
}
if classWithSameExtendedResourceName != nil {
switch cmp.Compare(newDeviceClass.CreationTimestamp.UnixNano(), classWithSameExtendedResourceName.CreationTimestamp.UnixNano()) {
case -1:
// New class is older, keep the current more recent one.
return
case 0:
// Equal, arbitrarily prefer "lower" name.
if newDeviceClass.Name >= classWithSameExtendedResourceName.Name {
return
}
}
}
// Remove old mappings first to handle ExtendedResourceName changes
if oldDeviceClass != nil {
delete(c.mapping, v1.ResourceName(resourceapi.ResourceDeviceClassPrefix+oldDeviceClass.Name))
@@ -149,14 +178,14 @@ func (c *ExtendedResourceCache) updateMapping(newDeviceClass, oldDeviceClass *re
// Add new mappings
if newDeviceClass.Spec.ExtendedResourceName != nil {
c.mapping[v1.ResourceName(*newDeviceClass.Spec.ExtendedResourceName)] = newDeviceClass.Name
c.mapping[v1.ResourceName(*newDeviceClass.Spec.ExtendedResourceName)] = newDeviceClass
c.logger.V(5).Info("Updated extended resource cache for explicit mapping",
"extendedResource", *newDeviceClass.Spec.ExtendedResourceName,
"deviceClass", newDeviceClass.Name)
}
// Always add the default mapping
defaultResourceName := v1.ResourceName(resourceapi.ResourceDeviceClassPrefix + newDeviceClass.Name)
c.mapping[defaultResourceName] = newDeviceClass.Name
c.mapping[defaultResourceName] = newDeviceClass
c.logger.V(5).Info("Updated extended resource cache for default mapping",
"extendedResource", defaultResourceName,
"deviceClass", newDeviceClass.Name)

View File

@@ -17,25 +17,29 @@ limitations under the License.
package extendedresourcecache
import (
"context"
"testing"
"time"
v1 "k8s.io/api/core/v1"
resourceapi "k8s.io/api/resource/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2/ktesting"
"k8s.io/utils/ptr"
)
type deviceClassResolver interface {
GetDeviceClass(resourceName v1.ResourceName) string
GetDeviceClass(resourceName v1.ResourceName) *resourceapi.DeviceClass
}
func TestNil(t *testing.T) {
var cache *ExtendedResourceCache
var resolver deviceClassResolver = cache
if className := resolver.GetDeviceClass("example.com/gpu"); className != "" {
t.Errorf("Expected the empty class name from a nil instance, got instead: %q", className)
if class := resolver.GetDeviceClass("example.com/gpu"); class != nil {
t.Errorf("Expected the nil class from a nil instance, got instead: %q", class.Name)
}
}
@@ -97,8 +101,8 @@ func TestHandlers(t *testing.T) {
if numAdd != 2 {
t.Errorf("second handler expected Add to be called last once, actual add #%d", numAdd)
}
if className := erCache.GetDeviceClass(resourceName); className != class.Name {
t.Errorf("expected %q, got %q", class.Name, className)
if deviceClass := erCache.GetDeviceClass(resourceName); deviceClass == nil || deviceClass.Name != class.Name {
t.Errorf("expected %q, got %q", class.Name, deviceClass)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
@@ -112,8 +116,8 @@ func TestHandlers(t *testing.T) {
if numUpdate != 2 {
t.Errorf("second handler expected Update to be called last once, actual update #%d", numUpdate)
}
if className := erCache.GetDeviceClass(resourceName); className != "" {
t.Errorf("expected %q, got %q", "", className)
if class := erCache.GetDeviceClass(resourceName); class != nil {
t.Errorf("expected %q, got %v", "", class)
}
},
DeleteFunc: func(obj interface{}) {
@@ -124,8 +128,8 @@ func TestHandlers(t *testing.T) {
if numDelete != 2 {
t.Errorf("second handler expected Delete to be called last once, actual delete #%d", numDelete)
}
if className := erCache.GetDeviceClass(resourceName); className != "" {
t.Errorf("expected %q, got %q", "", className)
if class := erCache.GetDeviceClass(resourceName); class != nil {
t.Errorf("expected %q, got %q", "", class)
}
},
}
@@ -137,13 +141,33 @@ func TestHandlers(t *testing.T) {
}
func TestExtendedResourceCache(t *testing.T) {
logger, _ := ktesting.NewTestContext(t)
logger, ctx := ktesting.NewTestContext(t)
tCtx, tCancel := context.WithCancel(ctx)
client := fake.NewClientset()
informerFactory := informers.NewSharedInformerFactory(client, 0)
deviceClassInformer := informerFactory.Resource().V1().DeviceClasses()
cache := NewExtendedResourceCache(logger)
if _, err := deviceClassInformer.Informer().AddEventHandler(cache); err != nil {
logger.Error(err, "Failed to add event handler for device classes")
}
informerFactory.Start(tCtx.Done())
t.Cleanup(func() {
// Need to cancel before waiting for the shutdown.
tCancel()
// Now we can wait for all goroutines to stop.
informerFactory.Shutdown()
})
informerFactory.WaitForCacheSync(tCtx.Done())
// Test with a device class that has an explicit extended resource name
now := time.Now()
deviceClass1 := &resourceapi.DeviceClass{
ObjectMeta: metav1.ObjectMeta{
Name: "gpu-class",
CreationTimestamp: metav1.Time{
Time: now,
},
},
Spec: resourceapi.DeviceClassSpec{
ExtendedResourceName: ptr.To("example.com/gpu"),
@@ -159,51 +183,143 @@ func TestExtendedResourceCache(t *testing.T) {
// No explicit extended resource name
},
}
deviceClass3 := &resourceapi.DeviceClass{
ObjectMeta: metav1.ObjectMeta{
Name: "gpu-class-3",
CreationTimestamp: metav1.Time{
Time: now.Add(-24 * time.Hour),
},
},
Spec: resourceapi.DeviceClassSpec{
ExtendedResourceName: ptr.To("example.com/gpu"),
},
}
deviceClass4 := &resourceapi.DeviceClass{
ObjectMeta: metav1.ObjectMeta{
Name: "gpu-class-4",
CreationTimestamp: metav1.Time{
Time: now.Add(time.Hour),
},
},
Spec: resourceapi.DeviceClassSpec{
ExtendedResourceName: ptr.To("example.com/gpu"),
},
}
deviceClass0 := &resourceapi.DeviceClass{
ObjectMeta: metav1.ObjectMeta{
Name: "gpu-class-0",
CreationTimestamp: metav1.Time{
Time: now.Add(time.Hour),
},
},
Spec: resourceapi.DeviceClassSpec{
ExtendedResourceName: ptr.To("example.com/gpu"),
},
}
// Test adding device classes
cache.OnAdd(deviceClass1, false)
cache.OnAdd(deviceClass2, false)
_, err := client.ResourceV1().DeviceClasses().Create(tCtx, deviceClass1, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Failed to create device class: %v", err)
}
_, err = client.ResourceV1().DeviceClasses().Create(tCtx, deviceClass2, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Failed to create device class: %v", err)
}
time.Sleep(1 * time.Second)
// Verify explicit mapping
deviceClassName := cache.GetDeviceClass("example.com/gpu")
if deviceClassName != "gpu-class" {
t.Errorf("Expected to find device class 'gpu-class' for 'example.com/gpu', got %s", deviceClassName)
deviceClass := cache.GetDeviceClass("example.com/gpu")
if deviceClass == nil || deviceClass.Name != "gpu-class" {
t.Errorf("Expected to find device class 'gpu-class' for 'example.com/gpu', got %v", deviceClass)
}
// Verify default mapping
defaultResourceName := v1.ResourceName("deviceclass.resource.kubernetes.io/fpga-class")
deviceClassName = cache.GetDeviceClass(defaultResourceName)
if deviceClassName != "fpga-class" {
t.Errorf("Expected to find device class 'fpga-class' for '%s', got %s", defaultResourceName, deviceClassName)
deviceClass = cache.GetDeviceClass(defaultResourceName)
if deviceClass == nil || deviceClass.Name != "fpga-class" {
t.Errorf("Expected to find device class 'fpga-class' for '%s', got %v", defaultResourceName, deviceClass)
}
// Verify both device classes have default mappings
if cache.GetDeviceClass("deviceclass.resource.kubernetes.io/gpu-class") != "gpu-class" {
deviceClass = cache.GetDeviceClass("deviceclass.resource.kubernetes.io/gpu-class")
if deviceClass == nil || deviceClass.Name != "gpu-class" {
t.Error("Expected default mapping for gpu-class")
}
// deviceClass3 is older than deviceClass1, hence it won't replace deviceClass1
_, err = client.ResourceV1().DeviceClasses().Create(tCtx, deviceClass3, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Failed to create device class: %v", err)
}
time.Sleep(1 * time.Second)
// should keep deviceClass1, since it is newer than deviceClass3
deviceClass = cache.GetDeviceClass("example.com/gpu")
if deviceClass == nil || deviceClass.Name != "gpu-class" {
t.Errorf("Expected to find device class 'gpu-class' for 'example.com/gpu', got %v", deviceClass)
}
// deviceClass4 is newer than deviceClass1, hence it will replace deviceClass1
_, err = client.ResourceV1().DeviceClasses().Create(tCtx, deviceClass4, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Failed to create device class: %v", err)
}
time.Sleep(1 * time.Second)
// deviceClass4 replaces deviceClass1, since it is newer with the same example.com/gpu extended resource name
deviceClass = cache.GetDeviceClass("example.com/gpu")
if deviceClass == nil || deviceClass.Name != "gpu-class-4" {
t.Errorf("Expected to find device class 'gpu-class' for 'example.com/gpu', got %v", deviceClass)
}
// deviceClass0 is created at the same time as deviceClass4, but its name is alphabetically ordered earlier,
// hence it will replace deviceClass4
_, err = client.ResourceV1().DeviceClasses().Create(tCtx, deviceClass0, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Failed to create device class: %v", err)
}
time.Sleep(1 * time.Second)
// deviceClass0 replaces deviceClass4, it is created at the same time as deviceClass4, but its name is
// alphabetically ordered earlier
deviceClass = cache.GetDeviceClass("example.com/gpu")
if deviceClass == nil || deviceClass.Name != "gpu-class-0" {
t.Errorf("Expected to find device class 'gpu-class' for 'example.com/gpu', got %v", deviceClass)
}
// Test modifying a device class
deviceClass1Modified := deviceClass1.DeepCopy()
deviceClass1Modified.Spec.ExtendedResourceName = ptr.To("test.com/gpu")
cache.OnUpdate(deviceClass1, deviceClass1Modified)
deviceClass0Modified := deviceClass0.DeepCopy()
deviceClass0Modified.Spec.ExtendedResourceName = ptr.To("test.com/gpu")
_, err = client.ResourceV1().DeviceClasses().Update(tCtx, deviceClass0Modified, metav1.UpdateOptions{})
if err != nil {
t.Fatalf("Failed to update device class: %v", err)
}
time.Sleep(1 * time.Second)
// Should have the new mapping
if cache.GetDeviceClass("test.com/gpu") != "gpu-class" {
t.Errorf("Expected to find device class 'gpu-class' for 'test.com/gpu' after modification, got %s", deviceClassName)
deviceClass = cache.GetDeviceClass("test.com/gpu")
if deviceClass == nil || deviceClass.Name != "gpu-class-0" {
t.Errorf("Expected to find device class 'gpu-class-0' for 'test.com/gpu' after modification, got %v", deviceClass)
}
// Should not have the old mapping for example.com/gpu
if cache.GetDeviceClass("example.com/gpu") != "" {
if cache.GetDeviceClass("example.com/gpu") != nil {
t.Errorf("Expected 'example.com/gpu' to be removed after modification, got %s", cache.GetDeviceClass("example.com/gpu"))
}
// Test deleting a device class
cache.OnDelete(deviceClass1Modified)
deviceClassName = cache.GetDeviceClass("test.com/gpu")
if deviceClassName != "" {
t.Errorf("Expected 'test.com/gpu' to be removed after deleting device class, got %s", deviceClassName)
err = client.ResourceV1().DeviceClasses().Delete(tCtx, deviceClass0.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Failed to delete device class: %v", err)
}
time.Sleep(1 * time.Second)
deviceClass = cache.GetDeviceClass("test.com/gpu")
if deviceClass != nil {
t.Errorf("Expected 'test.com/gpu' to be removed after deleting device class, got %s", deviceClass)
}
// Verify the default mapping is removed
if cache.GetDeviceClass("deviceclass.resource.kubernetes.io/gpu-class") != "" {
t.Errorf("Expected 'deviceclass.resource.kubernetes.io/gpu-class' to be removed after deleting device class, got %s", cache.GetDeviceClass("deviceclass.resource.kubernetes.io/gpu-class"))
if cache.GetDeviceClass("deviceclass.resource.kubernetes.io/gpu-class-0") != nil {
t.Errorf("Expected 'deviceclass.resource.kubernetes.io/gpu-class-0' to be removed after deleting device class, got %s", cache.GetDeviceClass("deviceclass.resource.kubernetes.io/gpu-class"))
}
}

View File

@@ -119,10 +119,10 @@ type ResourceClaimTracker interface {
// DeviceClassResolver resolves device class names from extended resource names.
type DeviceClassResolver interface {
// GetDeviceClass returns the device class name for the given extended resource name.
// Returns empty string if no mapping exists for the resource name or
// GetDeviceClass returns the device class for the given extended resource name.
// Returns nil if no mapping exists for the resource name or
// the DRAExtendedResource feature is disabled.
GetDeviceClass(resourceName v1.ResourceName) string
GetDeviceClass(resourceName v1.ResourceName) *resourceapi.DeviceClass
}
// SharedDRAManager can be used to obtain DRA objects, and track modifications to them in-memory - mainly by the DRA plugin.