Support cluster-level extended resources in kubelet and kube-scheduler

Co-authored-by: Yang Guo <ygg@google.com>
Co-authored-by: Chun Chen <chenchun.feed@gmail.com>

pkg/scheduler/algorithm/predicates/BUILD

@@ -61,6 +61,7 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )

pkg/scheduler/algorithm/predicates/metadata.go

@@ -22,6 +22,7 @@ import (
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/scheduler/algorithm"
 	"k8s.io/kubernetes/pkg/scheduler/schedulercache"
 	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
@@ -53,6 +54,13 @@ type predicateMetadata struct {
 	serviceAffinityInUse               bool
 	serviceAffinityMatchingPodList     []*v1.Pod
 	serviceAffinityMatchingPodServices []*v1.Service
+	// ignoredExtendedResources is a set of extended resource names that will
+	// be ignored in the PodFitsResources predicate.
+	//
+	// They can be scheduler extender managed resources, the consumption of
+	// which should be accounted only by the extenders. This set is synthesized
+	// from scheduler extender configuration and does not change per pod.
+	ignoredExtendedResources sets.String
 }
 
 // Ensure that predicateMetadata implements algorithm.PredicateMetadata.
@@ -71,6 +79,17 @@ func RegisterPredicateMetadataProducer(predicateName string, precomp PredicateMe
 	predicateMetadataProducers[predicateName] = precomp
 }
 
+// RegisterPredicateMetadataProducerWithExtendedResourceOptions registers a
+// PredicateMetadataProducer that creates predicate metadata with the provided
+// options for extended resources.
+//
+// See the comments in "predicateMetadata" for the explanation of the options.
+func RegisterPredicateMetadataProducerWithExtendedResourceOptions(ignoredExtendedResources sets.String) {
+	RegisterPredicateMetadataProducer("PredicateWithExtendedResourceOptions", func(pm *predicateMetadata) {
+		pm.ignoredExtendedResources = ignoredExtendedResources
+	})
+}
+
 // NewPredicateMetadataFactory creates a PredicateMetadataFactory.
 func NewPredicateMetadataFactory(podLister algorithm.PodLister) algorithm.PredicateMetadataProducer {
 	factory := &PredicateMetadataFactory{
@@ -170,10 +189,11 @@ func (meta *predicateMetadata) AddPod(addedPod *v1.Pod, nodeInfo *schedulercache
 // its maps and slices, but it does not copy the contents of pointer values.
 func (meta *predicateMetadata) ShallowCopy() algorithm.PredicateMetadata {
 	newPredMeta := &predicateMetadata{
-		pod:                  meta.pod,
-		podBestEffort:        meta.podBestEffort,
-		podRequest:           meta.podRequest,
-		serviceAffinityInUse: meta.serviceAffinityInUse,
+		pod:                      meta.pod,
+		podBestEffort:            meta.podBestEffort,
+		podRequest:               meta.podRequest,
+		serviceAffinityInUse:     meta.serviceAffinityInUse,
+		ignoredExtendedResources: meta.ignoredExtendedResources,
 	}
 	newPredMeta.podPorts = append([]*v1.ContainerPort(nil), meta.podPorts...)
 	newPredMeta.matchingAntiAffinityTerms = map[string][]matchingPodAntiAffinityTerm{}

pkg/scheduler/algorithm/predicates/predicates.go

@@ -29,6 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/apimachinery/pkg/util/sets"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	corelisters "k8s.io/client-go/listers/core/v1"
 	storagelisters "k8s.io/client-go/listers/storage/v1"
@@ -712,9 +713,15 @@ func PodFitsResources(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s
 		predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourcePods, 1, int64(len(nodeInfo.Pods())), int64(allowedPodNumber)))
 	}
 
+	// No extended resources should be ignored by default.
+	ignoredExtendedResources := sets.NewString()
+
 	var podRequest *schedulercache.Resource
 	if predicateMeta, ok := meta.(*predicateMetadata); ok {
 		podRequest = predicateMeta.podRequest
+		if predicateMeta.ignoredExtendedResources != nil {
+			ignoredExtendedResources = predicateMeta.ignoredExtendedResources
+		}
 	} else {
 		// We couldn't parse metadata - fallback to computing it.
 		podRequest = GetResourceRequest(pod)
@@ -743,6 +750,13 @@ func PodFitsResources(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s
 	}
 
 	for rName, rQuant := range podRequest.ScalarResources {
+		if v1helper.IsExtendedResourceName(rName) {
+			// If this resource is one of the extended resources that should be
+			// ignored, we will skip checking it.
+			if ignoredExtendedResources.Has(string(rName)) {
+				continue
+			}
+		}
 		if allocatable.ScalarResources[rName] < rQuant+nodeInfo.RequestedResource().ScalarResources[rName] {
 			predicateFails = append(predicateFails, NewInsufficientResourceError(rName, podRequest.ScalarResources[rName], nodeInfo.RequestedResource().ScalarResources[rName], allocatable.ScalarResources[rName]))
 		}

pkg/scheduler/algorithm/predicates/predicates_test.go

@@ -27,6 +27,7 @@ import (
 	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
@@ -93,11 +94,12 @@ func PredicateMetadata(p *v1.Pod, nodeInfo map[string]*schedulercache.NodeInfo)
 
 func TestPodFitsResources(t *testing.T) {
 	enoughPodsTests := []struct {
-		pod      *v1.Pod
-		nodeInfo *schedulercache.NodeInfo
-		fits     bool
-		test     string
-		reasons  []algorithm.PredicateFailureReason
+		pod                      *v1.Pod
+		nodeInfo                 *schedulercache.NodeInfo
+		fits                     bool
+		test                     string
+		reasons                  []algorithm.PredicateFailureReason
+		ignoredExtendedResources sets.String
 	}{
 		{
 			pod: &v1.Pod{},
@@ -323,12 +325,23 @@ func TestPodFitsResources(t *testing.T) {
 			test:    "hugepages resource allocatable enforced for multiple containers",
 			reasons: []algorithm.PredicateFailureReason{NewInsufficientResourceError(hugePageResourceA, 6, 2, 5)},
 		},
+		{
+			pod: newResourcePod(
+				schedulercache.Resource{MilliCPU: 1, Memory: 1, ScalarResources: map[v1.ResourceName]int64{extendedResourceB: 1}}),
+			nodeInfo: schedulercache.NewNodeInfo(
+				newResourcePod(schedulercache.Resource{MilliCPU: 0, Memory: 0})),
+			fits: true,
+			ignoredExtendedResources: sets.NewString(string(extendedResourceB)),
+			test: "skip checking ignored extended resource",
+		},
 	}
 
 	for _, test := range enoughPodsTests {
 		node := v1.Node{Status: v1.NodeStatus{Capacity: makeResources(10, 20, 0, 32, 5, 20, 5).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32, 5, 20, 5)}}
 		test.nodeInfo.SetNode(&node)
-		fits, reasons, err := PodFitsResources(test.pod, PredicateMetadata(test.pod, nil), test.nodeInfo)
+		RegisterPredicateMetadataProducerWithExtendedResourceOptions(test.ignoredExtendedResources)
+		meta := PredicateMetadata(test.pod, nil)
+		fits, reasons, err := PodFitsResources(test.pod, meta, test.nodeInfo)
 		if err != nil {
 			t.Errorf("%s: unexpected error: %v", test.test, err)
 		}