Merge pull request #50819 from NickrenREN/remove-overlay-scheduler

Automatic merge from submit-queue (batch tested with PRs 51235, 50819, 51274, 50972, 50504)

Changing scheduling part to manage one single local storage resource

**What this PR does / why we need it**:
 Finally decided to manage a single local storage resource

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*:  part of #50818

**Special notes for your reviewer**:
Since finally decided to manage a single local storage resource, remove overlay related code in scheduling part and change the name scratch to ephemeral storage.

**Release note**:
```release-note
Changing scheduling part of the alpha feature 'LocalStorageCapacityIsolation' to manage one single local ephemeral storage resource
```

/assign @jingxu97 
cc @ddysher

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -504,14 +504,6 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource {
 		result.Add(container.Resources.Requests)
 	}
 
-	// Account for storage requested by emptydir volumes
-	// If the storage medium is memory, should exclude the size
-	for _, vol := range pod.Spec.Volumes {
-		if vol.EmptyDir != nil && vol.EmptyDir.Medium != v1.StorageMediumMemory {
-			result.StorageScratch += vol.EmptyDir.SizeLimit.Value()
-		}
-	}
-
 	// take max_resource(sum_pod, any_init_container)
 	for _, container := range pod.Spec.InitContainers {
 		for rName, rQuantity := range container.Resources.Requests {
@@ -520,6 +512,10 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource {
 				if mem := rQuantity.Value(); mem > result.Memory {
 					result.Memory = mem
 				}
+			case v1.ResourceEphemeralStorage:
+				if ephemeralStorage := rQuantity.Value(); ephemeralStorage > result.EphemeralStorage {
+					result.EphemeralStorage = ephemeralStorage
+				}
 			case v1.ResourceCPU:
 				if cpu := rQuantity.MilliValue(); cpu > result.MilliCPU {
 					result.MilliCPU = cpu
@@ -528,10 +524,6 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource {
 				if gpu := rQuantity.Value(); gpu > result.NvidiaGPU {
 					result.NvidiaGPU = gpu
 				}
-			case v1.ResourceStorageOverlay:
-				if overlay := rQuantity.Value(); overlay > result.StorageOverlay {
-					result.StorageOverlay = overlay
-				}
 			default:
 				if v1helper.IsExtendedResourceName(rName) {
 					value := rQuantity.Value()
@@ -572,7 +564,7 @@ func PodFitsResources(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.No
 		// We couldn't parse metadata - fallback to computing it.
 		podRequest = GetResourceRequest(pod)
 	}
-	if podRequest.MilliCPU == 0 && podRequest.Memory == 0 && podRequest.NvidiaGPU == 0 && podRequest.StorageOverlay == 0 && podRequest.StorageScratch == 0 && len(podRequest.ExtendedResources) == 0 {
+	if podRequest.MilliCPU == 0 && podRequest.Memory == 0 && podRequest.NvidiaGPU == 0 && podRequest.EphemeralStorage == 0 && len(podRequest.ExtendedResources) == 0 {
 		return len(predicateFails) == 0, predicateFails, nil
 	}
 
@@ -587,20 +579,8 @@ func PodFitsResources(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.No
 		predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceNvidiaGPU, podRequest.NvidiaGPU, nodeInfo.RequestedResource().NvidiaGPU, allocatable.NvidiaGPU))
 	}
 
-	scratchSpaceRequest := podRequest.StorageScratch
-	if allocatable.StorageOverlay == 0 {
-		scratchSpaceRequest += podRequest.StorageOverlay
-		//scratchSpaceRequest += nodeInfo.RequestedResource().StorageOverlay
-		nodeScratchRequest := nodeInfo.RequestedResource().StorageOverlay + nodeInfo.RequestedResource().StorageScratch
-		if allocatable.StorageScratch < scratchSpaceRequest+nodeScratchRequest {
-			predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageScratch, scratchSpaceRequest, nodeScratchRequest, allocatable.StorageScratch))
-		}
-
-	} else if allocatable.StorageScratch < scratchSpaceRequest+nodeInfo.RequestedResource().StorageScratch {
-		predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageScratch, scratchSpaceRequest, nodeInfo.RequestedResource().StorageScratch, allocatable.StorageScratch))
-	}
-	if allocatable.StorageOverlay > 0 && allocatable.StorageOverlay < podRequest.StorageOverlay+nodeInfo.RequestedResource().StorageOverlay {
-		predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageOverlay, podRequest.StorageOverlay, nodeInfo.RequestedResource().StorageOverlay, allocatable.StorageOverlay))
+	if allocatable.EphemeralStorage < podRequest.EphemeralStorage+nodeInfo.RequestedResource().EphemeralStorage {
+		predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceEphemeralStorage, podRequest.EphemeralStorage, nodeInfo.RequestedResource().EphemeralStorage, allocatable.EphemeralStorage))
 	}
 
 	for rName, rQuant := range podRequest.ExtendedResources {

plugin/pkg/scheduler/algorithm/predicates/predicates_test.go

@@ -80,24 +80,24 @@ var (
 func makeResources(milliCPU, memory, nvidiaGPUs, pods, opaqueA, storage int64) v1.NodeResources {
 	return v1.NodeResources{
 		Capacity: v1.ResourceList{
-			v1.ResourceCPU:            *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
-			v1.ResourceMemory:         *resource.NewQuantity(memory, resource.BinarySI),
-			v1.ResourcePods:           *resource.NewQuantity(pods, resource.DecimalSI),
-			v1.ResourceNvidiaGPU:      *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI),
-			opaqueResourceA:           *resource.NewQuantity(opaqueA, resource.DecimalSI),
-			v1.ResourceStorageScratch: *resource.NewQuantity(storage, resource.BinarySI),
+			v1.ResourceCPU:              *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
+			v1.ResourceMemory:           *resource.NewQuantity(memory, resource.BinarySI),
+			v1.ResourcePods:             *resource.NewQuantity(pods, resource.DecimalSI),
+			v1.ResourceNvidiaGPU:        *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI),
+			opaqueResourceA:             *resource.NewQuantity(opaqueA, resource.DecimalSI),
+			v1.ResourceEphemeralStorage: *resource.NewQuantity(storage, resource.BinarySI),
 		},
 	}
 }
 
 func makeAllocatableResources(milliCPU, memory, nvidiaGPUs, pods, opaqueA, storage int64) v1.ResourceList {
 	return v1.ResourceList{
-		v1.ResourceCPU:            *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
-		v1.ResourceMemory:         *resource.NewQuantity(memory, resource.BinarySI),
-		v1.ResourcePods:           *resource.NewQuantity(pods, resource.DecimalSI),
-		v1.ResourceNvidiaGPU:      *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI),
-		opaqueResourceA:           *resource.NewQuantity(opaqueA, resource.DecimalSI),
-		v1.ResourceStorageScratch: *resource.NewQuantity(storage, resource.BinarySI),
+		v1.ResourceCPU:              *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
+		v1.ResourceMemory:           *resource.NewQuantity(memory, resource.BinarySI),
+		v1.ResourcePods:             *resource.NewQuantity(pods, resource.DecimalSI),
+		v1.ResourceNvidiaGPU:        *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI),
+		opaqueResourceA:             *resource.NewQuantity(opaqueA, resource.DecimalSI),
+		v1.ResourceEphemeralStorage: *resource.NewQuantity(storage, resource.BinarySI),
 	}
 }
 
@@ -421,92 +421,52 @@ func TestPodFitsResources(t *testing.T) {
 	}
 
 	storagePodsTests := []struct {
-		pod           *v1.Pod
-		emptyDirLimit int64
-		storageMedium v1.StorageMedium
-		nodeInfo      *schedulercache.NodeInfo
-		fits          bool
-		test          string
-		reasons       []algorithm.PredicateFailureReason
+		pod      *v1.Pod
+		nodeInfo *schedulercache.NodeInfo
+		fits     bool
+		test     string
+		reasons  []algorithm.PredicateFailureReason
 	}{
 		{
-			pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 1}),
+			pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1}),
 			nodeInfo: schedulercache.NewNodeInfo(
-				newResourcePod(schedulercache.Resource{MilliCPU: 10, Memory: 10, StorageOverlay: 20})),
+				newResourcePod(schedulercache.Resource{MilliCPU: 10, Memory: 10})),
 			fits: false,
 			test: "due to container scratch disk",
 			reasons: []algorithm.PredicateFailureReason{
 				NewInsufficientResourceError(v1.ResourceCPU, 1, 10, 10),
-				NewInsufficientResourceError(v1.ResourceStorageScratch, 1, 20, 20),
 			},
 		},
 		{
-			pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}),
+			pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1}),
 			nodeInfo: schedulercache.NewNodeInfo(
 				newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 10})),
 			fits: true,
 			test: "pod fit",
 		},
 		{
-			pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 18}),
+			pod: newResourcePod(schedulercache.Resource{EphemeralStorage: 25}),
 			nodeInfo: schedulercache.NewNodeInfo(
-				newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})),
+				newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2})),
 			fits: false,
-			test: "request exceeds allocatable overlay storage resource",
+			test: "storage ephemeral local storage request exceeds allocatable",
 			reasons: []algorithm.PredicateFailureReason{
-				NewInsufficientResourceError(v1.ResourceStorageScratch, 18, 5, 20),
+				NewInsufficientResourceError(v1.ResourceEphemeralStorage, 25, 0, 20),
 			},
 		},
 		{
-			pod: newResourcePod(schedulercache.Resource{StorageOverlay: 18}),
+			pod: newResourcePod(schedulercache.Resource{EphemeralStorage: 10}),
 			nodeInfo: schedulercache.NewNodeInfo(
-				newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})),
-			fits: false,
-			test: "request exceeds allocatable overlay storage resource",
-			reasons: []algorithm.PredicateFailureReason{
-				NewInsufficientResourceError(v1.ResourceStorageScratch, 18, 5, 20),
-			},
-		},
-		{
-			pod:           newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}),
-			emptyDirLimit: 15,
-			storageMedium: v1.StorageMediumDefault,
-			nodeInfo: schedulercache.NewNodeInfo(
-				newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})),
-			fits: false,
-			test: "storage scratchrequest exceeds allocatable",
-			reasons: []algorithm.PredicateFailureReason{
-				NewInsufficientResourceError(v1.ResourceStorageScratch, 25, 5, 20),
-			},
-		},
-		{
-			pod:           newResourcePod(schedulercache.Resource{}),
-			emptyDirLimit: 25,
-			storageMedium: v1.StorageMediumDefault,
-			nodeInfo: schedulercache.NewNodeInfo(
-				newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})),
-			fits: false,
-			test: "storage scratchrequest exceeds allocatable",
-			reasons: []algorithm.PredicateFailureReason{
-				NewInsufficientResourceError(v1.ResourceStorageScratch, 25, 5, 20),
-			},
-		},
-		{
-			pod:           newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}),
-			emptyDirLimit: 15,
-			storageMedium: v1.StorageMediumMemory,
-			nodeInfo: schedulercache.NewNodeInfo(
-				newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})),
+				newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2})),
 			fits: true,
-			test: "pod fit with memory medium",
+			test: "pod fits",
 		},
 	}
 
 	for _, test := range storagePodsTests {
 		node := v1.Node{Status: v1.NodeStatus{Capacity: makeResources(10, 20, 0, 32, 5, 20).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32, 5, 20)}}
 		test.nodeInfo.SetNode(&node)
-		pod := addStorageLimit(test.pod, test.emptyDirLimit, test.storageMedium)
-		fits, reasons, err := PodFitsResources(pod, PredicateMetadata(pod, nil), test.nodeInfo)
+		fits, reasons, err := PodFitsResources(test.pod, PredicateMetadata(test.pod, nil), test.nodeInfo)
 		if err != nil {
 			t.Errorf("%s: unexpected error: %v", test.test, err)
 		}

plugin/pkg/scheduler/schedulercache/node_info.go

@@ -63,11 +63,10 @@ type NodeInfo struct {
 
 // Resource is a collection of compute resource.
 type Resource struct {
-	MilliCPU       int64
-	Memory         int64
-	NvidiaGPU      int64
-	StorageScratch int64
-	StorageOverlay int64
+	MilliCPU         int64
+	Memory           int64
+	NvidiaGPU        int64
+	EphemeralStorage int64
 	// We store allowedPodNumber (which is Node.Status.Allocatable.Pods().Value())
 	// explicitly as int, to avoid conversions and improve performance.
 	AllowedPodNumber  int
@@ -97,10 +96,8 @@ func (r *Resource) Add(rl v1.ResourceList) {
 			r.NvidiaGPU += rQuant.Value()
 		case v1.ResourcePods:
 			r.AllowedPodNumber += int(rQuant.Value())
-		case v1.ResourceStorageScratch:
-			r.StorageScratch += rQuant.Value()
-		case v1.ResourceStorageOverlay:
-			r.StorageOverlay += rQuant.Value()
+		case v1.ResourceEphemeralStorage:
+			r.EphemeralStorage += rQuant.Value()
 		default:
 			if v1helper.IsExtendedResourceName(rName) {
 				r.AddExtended(rName, rQuant.Value())
@@ -111,12 +108,11 @@ func (r *Resource) Add(rl v1.ResourceList) {
 
 func (r *Resource) ResourceList() v1.ResourceList {
 	result := v1.ResourceList{
-		v1.ResourceCPU:            *resource.NewMilliQuantity(r.MilliCPU, resource.DecimalSI),
-		v1.ResourceMemory:         *resource.NewQuantity(r.Memory, resource.BinarySI),
-		v1.ResourceNvidiaGPU:      *resource.NewQuantity(r.NvidiaGPU, resource.DecimalSI),
-		v1.ResourcePods:           *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI),
-		v1.ResourceStorageOverlay: *resource.NewQuantity(r.StorageOverlay, resource.BinarySI),
-		v1.ResourceStorageScratch: *resource.NewQuantity(r.StorageScratch, resource.BinarySI),
+		v1.ResourceCPU:              *resource.NewMilliQuantity(r.MilliCPU, resource.DecimalSI),
+		v1.ResourceMemory:           *resource.NewQuantity(r.Memory, resource.BinarySI),
+		v1.ResourceNvidiaGPU:        *resource.NewQuantity(r.NvidiaGPU, resource.DecimalSI),
+		v1.ResourcePods:             *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI),
+		v1.ResourceEphemeralStorage: *resource.NewQuantity(r.EphemeralStorage, resource.BinarySI),
 	}
 	for rName, rQuant := range r.ExtendedResources {
 		result[rName] = *resource.NewQuantity(rQuant, resource.DecimalSI)
@@ -130,8 +126,7 @@ func (r *Resource) Clone() *Resource {
 		Memory:           r.Memory,
 		NvidiaGPU:        r.NvidiaGPU,
 		AllowedPodNumber: r.AllowedPodNumber,
-		StorageOverlay:   r.StorageOverlay,
-		StorageScratch:   r.StorageScratch,
+		EphemeralStorage: r.EphemeralStorage,
 	}
 	if r.ExtendedResources != nil {
 		res.ExtendedResources = make(map[v1.ResourceName]int64)
@@ -304,8 +299,7 @@ func (n *NodeInfo) addPod(pod *v1.Pod) {
 	n.requestedResource.MilliCPU += res.MilliCPU
 	n.requestedResource.Memory += res.Memory
 	n.requestedResource.NvidiaGPU += res.NvidiaGPU
-	n.requestedResource.StorageOverlay += res.StorageOverlay
-	n.requestedResource.StorageScratch += res.StorageScratch
+	n.requestedResource.EphemeralStorage += res.EphemeralStorage
 	if n.requestedResource.ExtendedResources == nil && len(res.ExtendedResources) > 0 {
 		n.requestedResource.ExtendedResources = map[v1.ResourceName]int64{}
 	}
@@ -392,14 +386,6 @@ func calculateResource(pod *v1.Pod) (res Resource, non0_cpu int64, non0_mem int6
 		// No non-zero resources for GPUs or opaque resources.
 	}
 
-	// Account for storage requested by emptydir volumes
-	// If the storage medium is memory, should exclude the size
-	for _, vol := range pod.Spec.Volumes {
-		if vol.EmptyDir != nil && vol.EmptyDir.Medium != v1.StorageMediumMemory {
-			res.StorageScratch += vol.EmptyDir.SizeLimit.Value()
-		}
-	}
-
 	return
 }