From c95ecbc7e8adb894b0daf9139551449a675c3a8c Mon Sep 17 00:00:00 2001 From: NickrenREN Date: Thu, 17 Aug 2017 09:44:15 +0800 Subject: [PATCH] Local storage does not manage overlay any more --- .../algorithm/predicates/predicates.go | 34 ++----- .../algorithm/predicates/predicates_test.go | 96 ++++++------------- .../pkg/scheduler/schedulercache/node_info.go | 40 +++----- 3 files changed, 48 insertions(+), 122 deletions(-) diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index d8e644b4349..a30232d22d3 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -504,14 +504,6 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource { result.Add(container.Resources.Requests) } - // Account for storage requested by emptydir volumes - // If the storage medium is memory, should exclude the size - for _, vol := range pod.Spec.Volumes { - if vol.EmptyDir != nil && vol.EmptyDir.Medium != v1.StorageMediumMemory { - result.StorageScratch += vol.EmptyDir.SizeLimit.Value() - } - } - // take max_resource(sum_pod, any_init_container) for _, container := range pod.Spec.InitContainers { for rName, rQuantity := range container.Resources.Requests { @@ -520,6 +512,10 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource { if mem := rQuantity.Value(); mem > result.Memory { result.Memory = mem } + case v1.ResourceEphemeralStorage: + if ephemeralStorage := rQuantity.Value(); ephemeralStorage > result.EphemeralStorage { + result.EphemeralStorage = ephemeralStorage + } case v1.ResourceCPU: if cpu := rQuantity.MilliValue(); cpu > result.MilliCPU { result.MilliCPU = cpu @@ -528,10 +524,6 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource { if gpu := rQuantity.Value(); gpu > result.NvidiaGPU { result.NvidiaGPU = gpu } - case v1.ResourceStorageOverlay: - if overlay := rQuantity.Value(); overlay > result.StorageOverlay { - result.StorageOverlay = overlay - } default: if v1helper.IsExtendedResourceName(rName) { value := rQuantity.Value() @@ -572,7 +564,7 @@ func PodFitsResources(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.No // We couldn't parse metadata - fallback to computing it. podRequest = GetResourceRequest(pod) } - if podRequest.MilliCPU == 0 && podRequest.Memory == 0 && podRequest.NvidiaGPU == 0 && podRequest.StorageOverlay == 0 && podRequest.StorageScratch == 0 && len(podRequest.ExtendedResources) == 0 { + if podRequest.MilliCPU == 0 && podRequest.Memory == 0 && podRequest.NvidiaGPU == 0 && podRequest.EphemeralStorage == 0 && len(podRequest.ExtendedResources) == 0 { return len(predicateFails) == 0, predicateFails, nil } @@ -587,20 +579,8 @@ func PodFitsResources(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.No predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceNvidiaGPU, podRequest.NvidiaGPU, nodeInfo.RequestedResource().NvidiaGPU, allocatable.NvidiaGPU)) } - scratchSpaceRequest := podRequest.StorageScratch - if allocatable.StorageOverlay == 0 { - scratchSpaceRequest += podRequest.StorageOverlay - //scratchSpaceRequest += nodeInfo.RequestedResource().StorageOverlay - nodeScratchRequest := nodeInfo.RequestedResource().StorageOverlay + nodeInfo.RequestedResource().StorageScratch - if allocatable.StorageScratch < scratchSpaceRequest+nodeScratchRequest { - predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageScratch, scratchSpaceRequest, nodeScratchRequest, allocatable.StorageScratch)) - } - - } else if allocatable.StorageScratch < scratchSpaceRequest+nodeInfo.RequestedResource().StorageScratch { - predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageScratch, scratchSpaceRequest, nodeInfo.RequestedResource().StorageScratch, allocatable.StorageScratch)) - } - if allocatable.StorageOverlay > 0 && allocatable.StorageOverlay < podRequest.StorageOverlay+nodeInfo.RequestedResource().StorageOverlay { - predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageOverlay, podRequest.StorageOverlay, nodeInfo.RequestedResource().StorageOverlay, allocatable.StorageOverlay)) + if allocatable.EphemeralStorage < podRequest.EphemeralStorage+nodeInfo.RequestedResource().EphemeralStorage { + predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceEphemeralStorage, podRequest.EphemeralStorage, nodeInfo.RequestedResource().EphemeralStorage, allocatable.EphemeralStorage)) } for rName, rQuant := range podRequest.ExtendedResources { diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go index 90c7be86ab8..d592dfe0338 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -80,24 +80,24 @@ var ( func makeResources(milliCPU, memory, nvidiaGPUs, pods, opaqueA, storage int64) v1.NodeResources { return v1.NodeResources{ Capacity: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), - v1.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), - v1.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), - opaqueResourceA: *resource.NewQuantity(opaqueA, resource.DecimalSI), - v1.ResourceStorageScratch: *resource.NewQuantity(storage, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), + v1.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), + opaqueResourceA: *resource.NewQuantity(opaqueA, resource.DecimalSI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(storage, resource.BinarySI), }, } } func makeAllocatableResources(milliCPU, memory, nvidiaGPUs, pods, opaqueA, storage int64) v1.ResourceList { return v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), - v1.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), - v1.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), - opaqueResourceA: *resource.NewQuantity(opaqueA, resource.DecimalSI), - v1.ResourceStorageScratch: *resource.NewQuantity(storage, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), + v1.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), + opaqueResourceA: *resource.NewQuantity(opaqueA, resource.DecimalSI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(storage, resource.BinarySI), } } @@ -421,92 +421,52 @@ func TestPodFitsResources(t *testing.T) { } storagePodsTests := []struct { - pod *v1.Pod - emptyDirLimit int64 - storageMedium v1.StorageMedium - nodeInfo *schedulercache.NodeInfo - fits bool - test string - reasons []algorithm.PredicateFailureReason + pod *v1.Pod + nodeInfo *schedulercache.NodeInfo + fits bool + test string + reasons []algorithm.PredicateFailureReason }{ { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 1}), + pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1}), nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 10, Memory: 10, StorageOverlay: 20})), + newResourcePod(schedulercache.Resource{MilliCPU: 10, Memory: 10})), fits: false, test: "due to container scratch disk", reasons: []algorithm.PredicateFailureReason{ NewInsufficientResourceError(v1.ResourceCPU, 1, 10, 10), - NewInsufficientResourceError(v1.ResourceStorageScratch, 1, 20, 20), }, }, { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}), + pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1}), nodeInfo: schedulercache.NewNodeInfo( newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 10})), fits: true, test: "pod fit", }, { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 18}), + pod: newResourcePod(schedulercache.Resource{EphemeralStorage: 25}), nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), + newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2})), fits: false, - test: "request exceeds allocatable overlay storage resource", + test: "storage ephemeral local storage request exceeds allocatable", reasons: []algorithm.PredicateFailureReason{ - NewInsufficientResourceError(v1.ResourceStorageScratch, 18, 5, 20), + NewInsufficientResourceError(v1.ResourceEphemeralStorage, 25, 0, 20), }, }, { - pod: newResourcePod(schedulercache.Resource{StorageOverlay: 18}), + pod: newResourcePod(schedulercache.Resource{EphemeralStorage: 10}), nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), - fits: false, - test: "request exceeds allocatable overlay storage resource", - reasons: []algorithm.PredicateFailureReason{ - NewInsufficientResourceError(v1.ResourceStorageScratch, 18, 5, 20), - }, - }, - { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}), - emptyDirLimit: 15, - storageMedium: v1.StorageMediumDefault, - nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), - fits: false, - test: "storage scratchrequest exceeds allocatable", - reasons: []algorithm.PredicateFailureReason{ - NewInsufficientResourceError(v1.ResourceStorageScratch, 25, 5, 20), - }, - }, - { - pod: newResourcePod(schedulercache.Resource{}), - emptyDirLimit: 25, - storageMedium: v1.StorageMediumDefault, - nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), - fits: false, - test: "storage scratchrequest exceeds allocatable", - reasons: []algorithm.PredicateFailureReason{ - NewInsufficientResourceError(v1.ResourceStorageScratch, 25, 5, 20), - }, - }, - { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}), - emptyDirLimit: 15, - storageMedium: v1.StorageMediumMemory, - nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), + newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2})), fits: true, - test: "pod fit with memory medium", + test: "pod fits", }, } for _, test := range storagePodsTests { node := v1.Node{Status: v1.NodeStatus{Capacity: makeResources(10, 20, 0, 32, 5, 20).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32, 5, 20)}} test.nodeInfo.SetNode(&node) - pod := addStorageLimit(test.pod, test.emptyDirLimit, test.storageMedium) - fits, reasons, err := PodFitsResources(pod, PredicateMetadata(pod, nil), test.nodeInfo) + fits, reasons, err := PodFitsResources(test.pod, PredicateMetadata(test.pod, nil), test.nodeInfo) if err != nil { t.Errorf("%s: unexpected error: %v", test.test, err) } diff --git a/plugin/pkg/scheduler/schedulercache/node_info.go b/plugin/pkg/scheduler/schedulercache/node_info.go index 6aa97213e01..c60a047cca4 100644 --- a/plugin/pkg/scheduler/schedulercache/node_info.go +++ b/plugin/pkg/scheduler/schedulercache/node_info.go @@ -63,11 +63,10 @@ type NodeInfo struct { // Resource is a collection of compute resource. type Resource struct { - MilliCPU int64 - Memory int64 - NvidiaGPU int64 - StorageScratch int64 - StorageOverlay int64 + MilliCPU int64 + Memory int64 + NvidiaGPU int64 + EphemeralStorage int64 // We store allowedPodNumber (which is Node.Status.Allocatable.Pods().Value()) // explicitly as int, to avoid conversions and improve performance. AllowedPodNumber int @@ -97,10 +96,8 @@ func (r *Resource) Add(rl v1.ResourceList) { r.NvidiaGPU += rQuant.Value() case v1.ResourcePods: r.AllowedPodNumber += int(rQuant.Value()) - case v1.ResourceStorageScratch: - r.StorageScratch += rQuant.Value() - case v1.ResourceStorageOverlay: - r.StorageOverlay += rQuant.Value() + case v1.ResourceEphemeralStorage: + r.EphemeralStorage += rQuant.Value() default: if v1helper.IsExtendedResourceName(rName) { r.AddExtended(rName, rQuant.Value()) @@ -111,12 +108,11 @@ func (r *Resource) Add(rl v1.ResourceList) { func (r *Resource) ResourceList() v1.ResourceList { result := v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(r.MilliCPU, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(r.Memory, resource.BinarySI), - v1.ResourceNvidiaGPU: *resource.NewQuantity(r.NvidiaGPU, resource.DecimalSI), - v1.ResourcePods: *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI), - v1.ResourceStorageOverlay: *resource.NewQuantity(r.StorageOverlay, resource.BinarySI), - v1.ResourceStorageScratch: *resource.NewQuantity(r.StorageScratch, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(r.MilliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(r.Memory, resource.BinarySI), + v1.ResourceNvidiaGPU: *resource.NewQuantity(r.NvidiaGPU, resource.DecimalSI), + v1.ResourcePods: *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(r.EphemeralStorage, resource.BinarySI), } for rName, rQuant := range r.ExtendedResources { result[rName] = *resource.NewQuantity(rQuant, resource.DecimalSI) @@ -130,8 +126,7 @@ func (r *Resource) Clone() *Resource { Memory: r.Memory, NvidiaGPU: r.NvidiaGPU, AllowedPodNumber: r.AllowedPodNumber, - StorageOverlay: r.StorageOverlay, - StorageScratch: r.StorageScratch, + EphemeralStorage: r.EphemeralStorage, } if r.ExtendedResources != nil { res.ExtendedResources = make(map[v1.ResourceName]int64) @@ -304,8 +299,7 @@ func (n *NodeInfo) addPod(pod *v1.Pod) { n.requestedResource.MilliCPU += res.MilliCPU n.requestedResource.Memory += res.Memory n.requestedResource.NvidiaGPU += res.NvidiaGPU - n.requestedResource.StorageOverlay += res.StorageOverlay - n.requestedResource.StorageScratch += res.StorageScratch + n.requestedResource.EphemeralStorage += res.EphemeralStorage if n.requestedResource.ExtendedResources == nil && len(res.ExtendedResources) > 0 { n.requestedResource.ExtendedResources = map[v1.ResourceName]int64{} } @@ -392,14 +386,6 @@ func calculateResource(pod *v1.Pod) (res Resource, non0_cpu int64, non0_mem int6 // No non-zero resources for GPUs or opaque resources. } - // Account for storage requested by emptydir volumes - // If the storage medium is memory, should exclude the size - for _, vol := range pod.Spec.Volumes { - if vol.EmptyDir != nil && vol.EmptyDir.Medium != v1.StorageMediumMemory { - res.StorageScratch += vol.EmptyDir.SizeLimit.Value() - } - } - return }