From a0a9ccfa87683a1414fd74e557d2abd920c1ee62 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Wed, 23 May 2018 12:49:16 -0400 Subject: [PATCH] Implement scheduler changes for volume limits --- pkg/scheduler/algorithm/predicates/BUILD | 4 + .../max_attachable_volume_predicate_test.go | 854 ++++++++++++++++++ .../algorithm/predicates/predicates.go | 31 +- .../algorithm/predicates/predicates_test.go | 773 ---------------- pkg/scheduler/cache/node_info.go | 11 + pkg/scheduler/cache/node_info_test.go | 7 +- 6 files changed, 897 insertions(+), 783 deletions(-) create mode 100644 pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go diff --git a/pkg/scheduler/algorithm/predicates/BUILD b/pkg/scheduler/algorithm/predicates/BUILD index bf8e46796d0..13e216174e6 100644 --- a/pkg/scheduler/algorithm/predicates/BUILD +++ b/pkg/scheduler/algorithm/predicates/BUILD @@ -46,6 +46,7 @@ go_library( go_test( name = "go_default_test", srcs = [ + "max_attachable_volume_predicate_test.go", "metadata_test.go", "predicates_test.go", "utils_test.go", @@ -53,10 +54,12 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core/v1/helper:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/cache:go_default_library", "//pkg/scheduler/testing:go_default_library", + "//pkg/volume/util:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", @@ -64,6 +67,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go new file mode 100644 index 00000000000..60aa5879a30 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go @@ -0,0 +1,854 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package predicates + +import ( + "os" + "reflect" + "strconv" + "strings" + "testing" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/scheduler/algorithm" + schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +func onePVCPod(filterName string) *v1.Pod { + return &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "some" + filterName + "Vol", + }, + }, + }, + }, + }, + } +} + +func splitPVCPod(filterName string) *v1.Pod { + return &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "someNon" + filterName + "Vol", + }, + }, + }, + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "some" + filterName + "Vol", + }, + }, + }, + }, + }, + } +} + +func TestVolumeCountConflicts(t *testing.T) { + oneVolPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp"}, + }, + }, + }, + }, + } + twoVolPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp1"}, + }, + }, + { + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp2"}, + }, + }, + }, + }, + } + splitVolsPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{}, + }, + }, + { + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "svp"}, + }, + }, + }, + }, + } + nonApplicablePod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{}, + }, + }, + }, + }, + } + deletedPVCPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "deletedPVC", + }, + }, + }, + }, + }, + } + twoDeletedPVCPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "deletedPVC", + }, + }, + }, + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "anotherDeletedPVC", + }, + }, + }, + }, + }, + } + deletedPVPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvcWithDeletedPV", + }, + }, + }, + }, + }, + } + // deletedPVPod2 is a different pod than deletedPVPod but using the same PVC + deletedPVPod2 := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvcWithDeletedPV", + }, + }, + }, + }, + }, + } + // anotherDeletedPVPod is a different pod than deletedPVPod and uses another PVC + anotherDeletedPVPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "anotherPVCWithDeletedPV", + }, + }, + }, + }, + }, + } + emptyPod := &v1.Pod{ + Spec: v1.PodSpec{}, + } + unboundPVCPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "unboundPVC", + }, + }, + }, + }, + }, + } + // Different pod than unboundPVCPod, but using the same unbound PVC + unboundPVCPod2 := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "unboundPVC", + }, + }, + }, + }, + }, + } + + // pod with unbound PVC that's different to unboundPVC + anotherUnboundPVCPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "anotherUnboundPVC", + }, + }, + }, + }, + }, + } + + tests := []struct { + newPod *v1.Pod + existingPods []*v1.Pod + filterName string + maxVols int + fits bool + test string + }{ + // filterName:EBSVolumeFilterType + { + newPod: oneVolPod, + existingPods: []*v1.Pod{twoVolPod, oneVolPod}, + filterName: EBSVolumeFilterType, + maxVols: 4, + fits: true, + test: "fits when node capacity >= new pod's EBS volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "doesn't fit when node capacity < new pod's EBS volumes", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{twoVolPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores non-EBS volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts ignore non-EBS volumes", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count considers PVCs backed by EBS volumes", + }, + { + newPod: splitPVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores PVCs not backed by EBS volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, onePVCPod(EBSVolumeFilterType)}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: false, + test: "existing pods' counts considers PVCs backed by EBS volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(EBSVolumeFilterType)}, + filterName: EBSVolumeFilterType, + maxVols: 4, + fits: true, + test: "already-mounted EBS volumes are always ok to allow", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(EBSVolumeFilterType)}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "the same EBS volumes are not counted multiple times", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: false, + test: "pod with missing two PVCs is counted towards the PV limit twice", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: deletedPVPod2, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing the same PV are counted towards the PV limit only once", + }, + { + newPod: anotherDeletedPVPod, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "two pods missing different PVs are counted towards the PV limit twice", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: unboundPVCPod2, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: true, + test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", + }, + { + newPod: anotherUnboundPVCPod, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "two different unbound PVCs are counted towards the PV limit as two volumes", + }, + // filterName:GCEPDVolumeFilterType + { + newPod: oneVolPod, + existingPods: []*v1.Pod{twoVolPod, oneVolPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 4, + fits: true, + test: "fits when node capacity >= new pod's GCE volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "fit when node capacity < new pod's GCE volumes", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{twoVolPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores non-GCE volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts ignore non-GCE volumes", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count considers PVCs backed by GCE volumes", + }, + { + newPod: splitPVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores PVCs not backed by GCE volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, onePVCPod(GCEPDVolumeFilterType)}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts considers PVCs backed by GCE volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(GCEPDVolumeFilterType)}, + filterName: GCEPDVolumeFilterType, + maxVols: 4, + fits: true, + test: "already-mounted EBS volumes are always ok to allow", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(GCEPDVolumeFilterType)}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "the same GCE volumes are not counted multiple times", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing two PVCs is counted towards the PV limit twice", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: deletedPVPod2, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing the same PV are counted towards the PV limit only once", + }, + { + newPod: anotherDeletedPVPod, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing different PVs are counted towards the PV limit twice", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: unboundPVCPod2, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", + }, + { + newPod: anotherUnboundPVCPod, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "two different unbound PVCs are counted towards the PV limit as two volumes", + }, + // filterName:AzureDiskVolumeFilterType + { + newPod: oneVolPod, + existingPods: []*v1.Pod{twoVolPod, oneVolPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 4, + fits: true, + test: "fits when node capacity >= new pod's AzureDisk volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "fit when node capacity < new pod's AzureDisk volumes", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{twoVolPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores non-AzureDisk volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts ignore non-AzureDisk volumes", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count considers PVCs backed by AzureDisk volumes", + }, + { + newPod: splitPVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores PVCs not backed by AzureDisk volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, onePVCPod(AzureDiskVolumeFilterType)}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts considers PVCs backed by AzureDisk volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(AzureDiskVolumeFilterType)}, + filterName: AzureDiskVolumeFilterType, + maxVols: 4, + fits: true, + test: "already-mounted AzureDisk volumes are always ok to allow", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(AzureDiskVolumeFilterType)}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "the same AzureDisk volumes are not counted multiple times", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing two PVCs is counted towards the PV limit twice", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: deletedPVPod2, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing the same PV are counted towards the PV limit only once", + }, + { + newPod: anotherDeletedPVPod, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing different PVs are counted towards the PV limit twice", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: unboundPVCPod2, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", + }, + { + newPod: anotherUnboundPVCPod, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "two different unbound PVCs are counted towards the PV limit as two volumes", + }, + } + + pvInfo := func(filterName string) FakePersistentVolumeInfo { + return FakePersistentVolumeInfo{ + { + ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"}, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: strings.ToLower(filterName) + "Vol"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"}, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{}, + }, + }, + } + } + + pvcInfo := func(filterName string) FakePersistentVolumeClaimInfo { + return FakePersistentVolumeClaimInfo{ + { + ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "some" + filterName + "Vol"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someNon" + filterName + "Vol"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pvcWithDeletedPV"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "pvcWithDeletedPV"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "anotherPVCWithDeletedPV"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "anotherPVCWithDeletedPV"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "unboundPVC"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: ""}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "anotherUnboundPVC"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: ""}, + }, + } + } + + expectedFailureReasons := []algorithm.PredicateFailureReason{ErrMaxVolumeCountExceeded} + + // running attachable predicate tests without feature gate and no limit present on nodes + for _, test := range tests { + os.Setenv(KubeMaxPDVols, strconv.Itoa(test.maxVols)) + pred := NewMaxPDVolumeCountPredicate(test.filterName, pvInfo(test.filterName), pvcInfo(test.filterName)) + fits, reasons, err := pred(test.newPod, PredicateMetadata(test.newPod, nil), schedulercache.NewNodeInfo(test.existingPods...)) + if err != nil { + t.Errorf("[%s]%s: unexpected error: %v", test.filterName, test.test, err) + } + if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { + t.Errorf("[%s]%s: unexpected failure reasons: %v, want: %v", test.filterName, test.test, reasons, expectedFailureReasons) + } + if fits != test.fits { + t.Errorf("[%s]%s: expected %v, got %v", test.filterName, test.test, test.fits, fits) + } + } + + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() + + // running attachable predicate tests with feature gate and limit present on nodes + for _, test := range tests { + node := getNodeWithPodAndVolumeLimits(test.existingPods, int64(test.maxVols), test.filterName) + pred := NewMaxPDVolumeCountPredicate(test.filterName, pvInfo(test.filterName), pvcInfo(test.filterName)) + fits, reasons, err := pred(test.newPod, PredicateMetadata(test.newPod, nil), node) + if err != nil { + t.Errorf("Using allocatable [%s]%s: unexpected error: %v", test.filterName, test.test, err) + } + if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { + t.Errorf("Using allocatable [%s]%s: unexpected failure reasons: %v, want: %v", test.filterName, test.test, reasons, expectedFailureReasons) + } + if fits != test.fits { + t.Errorf("Using allocatable [%s]%s: expected %v, got %v", test.filterName, test.test, test.fits, fits) + } + } +} + +func getNodeWithPodAndVolumeLimits(pods []*v1.Pod, limit int64, filter string) *schedulercache.NodeInfo { + nodeInfo := schedulercache.NewNodeInfo(pods...) + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-for-max-pd-test-1"}, + Status: v1.NodeStatus{ + Allocatable: v1.ResourceList{ + getVolumeLimitKey(filter): *resource.NewQuantity(limit, resource.DecimalSI), + }, + }, + } + nodeInfo.SetNode(node) + return nodeInfo +} + +func getVolumeLimitKey(filterType string) v1.ResourceName { + switch filterType { + case EBSVolumeFilterType: + return v1.ResourceName(volumeutil.EBSVolumeLimitKey) + case GCEPDVolumeFilterType: + return v1.ResourceName(volumeutil.GCEVolumeLimitKey) + case AzureDiskVolumeFilterType: + return v1.ResourceName(volumeutil.AzureVolumeLimitKey) + default: + return "" + } +} diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 71a4688a4d6..aaf28f9622e 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -289,10 +289,11 @@ func NoDiskConflict(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *sch // MaxPDVolumeCountChecker contains information to check the max number of volumes for a predicate. type MaxPDVolumeCountChecker struct { - filter VolumeFilter - maxVolumes int - pvInfo PersistentVolumeInfo - pvcInfo PersistentVolumeClaimInfo + filter VolumeFilter + volumeLimitKey v1.ResourceName + maxVolumes int + pvInfo PersistentVolumeInfo + pvcInfo PersistentVolumeClaimInfo // The string below is generated randomly during the struct's initialization. // It is used to prefix volumeID generated inside the predicate() method to @@ -313,21 +314,25 @@ type VolumeFilter struct { // The predicate looks for both volumes used directly, as well as PVC volumes that are backed by relevant volume // types, counts the number of unique volumes, and rejects the new pod if it would place the total count over // the maximum. -func NewMaxPDVolumeCountPredicate(filterName string, pvInfo PersistentVolumeInfo, pvcInfo PersistentVolumeClaimInfo) algorithm.FitPredicate { - +func NewMaxPDVolumeCountPredicate( + filterName string, pvInfo PersistentVolumeInfo, pvcInfo PersistentVolumeClaimInfo) algorithm.FitPredicate { var filter VolumeFilter var maxVolumes int + var volumeLimitKey v1.ResourceName switch filterName { case EBSVolumeFilterType: filter = EBSVolumeFilter + volumeLimitKey = v1.ResourceName(volumeutil.EBSVolumeLimitKey) maxVolumes = getMaxVols(DefaultMaxEBSVolumes) case GCEPDVolumeFilterType: filter = GCEPDVolumeFilter + volumeLimitKey = v1.ResourceName(volumeutil.GCEVolumeLimitKey) maxVolumes = getMaxVols(DefaultMaxGCEPDVolumes) case AzureDiskVolumeFilterType: filter = AzureDiskVolumeFilter + volumeLimitKey = v1.ResourceName(volumeutil.AzureVolumeLimitKey) maxVolumes = getMaxVols(DefaultMaxAzureDiskVolumes) default: glog.Fatalf("Wrong filterName, Only Support %v %v %v ", EBSVolumeFilterType, @@ -337,6 +342,7 @@ func NewMaxPDVolumeCountPredicate(filterName string, pvInfo PersistentVolumeInfo } c := &MaxPDVolumeCountChecker{ filter: filter, + volumeLimitKey: volumeLimitKey, maxVolumes: maxVolumes, pvInfo: pvInfo, pvcInfo: pvcInfo, @@ -362,7 +368,6 @@ func getMaxVols(defaultVal int) int { } func (c *MaxPDVolumeCountChecker) filterVolumes(volumes []v1.Volume, namespace string, filteredVolumes map[string]bool) error { - for i := range volumes { vol := &volumes[i] if id, ok := c.filter.FilterVolume(vol); ok { @@ -449,15 +454,23 @@ func (c *MaxPDVolumeCountChecker) predicate(pod *v1.Pod, meta algorithm.Predicat } numNewVolumes := len(newVolumes) + maxAttachLimit := c.maxVolumes - if numExistingVolumes+numNewVolumes > c.maxVolumes { + if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { + volumeLimits := nodeInfo.VolumeLimits() + if maxAttachLimitFromAllocatable, ok := volumeLimits[c.volumeLimitKey]; ok { + maxAttachLimit = int(maxAttachLimitFromAllocatable) + } + } + + if numExistingVolumes+numNewVolumes > maxAttachLimit { // violates MaxEBSVolumeCount or MaxGCEPDVolumeCount return false, []algorithm.PredicateFailureReason{ErrMaxVolumeCountExceeded}, nil } if nodeInfo != nil && nodeInfo.TransientInfo != nil && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) { nodeInfo.TransientInfo.TransientLock.Lock() defer nodeInfo.TransientInfo.TransientLock.Unlock() - nodeInfo.TransientInfo.TransNodeInfo.AllocatableVolumesCount = c.maxVolumes - numExistingVolumes + nodeInfo.TransientInfo.TransNodeInfo.AllocatableVolumesCount = maxAttachLimit - numExistingVolumes nodeInfo.TransientInfo.TransNodeInfo.RequestedVolumes = numNewVolumes } return true, nil, nil diff --git a/pkg/scheduler/algorithm/predicates/predicates_test.go b/pkg/scheduler/algorithm/predicates/predicates_test.go index e6d2868d478..b664590f363 100644 --- a/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -1833,779 +1833,6 @@ func TestServiceAffinity(t *testing.T) { } } -func onePVCPod(filterName string) *v1.Pod { - return &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "some" + filterName + "Vol", - }, - }, - }, - }, - }, - } -} - -func splitPVCPod(filterName string) *v1.Pod { - return &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "someNon" + filterName + "Vol", - }, - }, - }, - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "some" + filterName + "Vol", - }, - }, - }, - }, - }, - } -} - -func TestVolumeCountConflicts(t *testing.T) { - oneVolPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp"}, - }, - }, - }, - }, - } - twoVolPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp1"}, - }, - }, - { - VolumeSource: v1.VolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp2"}, - }, - }, - }, - }, - } - splitVolsPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{}, - }, - }, - { - VolumeSource: v1.VolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "svp"}, - }, - }, - }, - }, - } - nonApplicablePod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{}, - }, - }, - }, - }, - } - deletedPVCPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "deletedPVC", - }, - }, - }, - }, - }, - } - twoDeletedPVCPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "deletedPVC", - }, - }, - }, - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "anotherDeletedPVC", - }, - }, - }, - }, - }, - } - deletedPVPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "pvcWithDeletedPV", - }, - }, - }, - }, - }, - } - // deletedPVPod2 is a different pod than deletedPVPod but using the same PVC - deletedPVPod2 := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "pvcWithDeletedPV", - }, - }, - }, - }, - }, - } - // anotherDeletedPVPod is a different pod than deletedPVPod and uses another PVC - anotherDeletedPVPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "anotherPVCWithDeletedPV", - }, - }, - }, - }, - }, - } - emptyPod := &v1.Pod{ - Spec: v1.PodSpec{}, - } - unboundPVCPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "unboundPVC", - }, - }, - }, - }, - }, - } - // Different pod than unboundPVCPod, but using the same unbound PVC - unboundPVCPod2 := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "unboundPVC", - }, - }, - }, - }, - }, - } - - // pod with unbound PVC that's different to unboundPVC - anotherUnboundPVCPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "anotherUnboundPVC", - }, - }, - }, - }, - }, - } - - tests := []struct { - newPod *v1.Pod - existingPods []*v1.Pod - filterName string - maxVols int - fits bool - test string - }{ - // filterName:EBSVolumeFilterType - { - newPod: oneVolPod, - existingPods: []*v1.Pod{twoVolPod, oneVolPod}, - filterName: EBSVolumeFilterType, - maxVols: 4, - fits: true, - test: "fits when node capacity >= new pod's EBS volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "doesn't fit when node capacity < new pod's EBS volumes", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{twoVolPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores non-EBS volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts ignore non-EBS volumes", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count considers PVCs backed by EBS volumes", - }, - { - newPod: splitPVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores PVCs not backed by EBS volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, onePVCPod(EBSVolumeFilterType)}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: false, - test: "existing pods' counts considers PVCs backed by EBS volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(EBSVolumeFilterType)}, - filterName: EBSVolumeFilterType, - maxVols: 4, - fits: true, - test: "already-mounted EBS volumes are always ok to allow", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(EBSVolumeFilterType)}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "the same EBS volumes are not counted multiple times", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: false, - test: "pod with missing two PVCs is counted towards the PV limit twice", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: deletedPVPod2, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing the same PV are counted towards the PV limit only once", - }, - { - newPod: anotherDeletedPVPod, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "two pods missing different PVs are counted towards the PV limit twice", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: unboundPVCPod2, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: true, - test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", - }, - { - newPod: anotherUnboundPVCPod, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "two different unbound PVCs are counted towards the PV limit as two volumes", - }, - // filterName:GCEPDVolumeFilterType - { - newPod: oneVolPod, - existingPods: []*v1.Pod{twoVolPod, oneVolPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 4, - fits: true, - test: "fits when node capacity >= new pod's GCE volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "fit when node capacity < new pod's GCE volumes", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{twoVolPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores non-GCE volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts ignore non-GCE volumes", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count considers PVCs backed by GCE volumes", - }, - { - newPod: splitPVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores PVCs not backed by GCE volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, onePVCPod(GCEPDVolumeFilterType)}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts considers PVCs backed by GCE volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(GCEPDVolumeFilterType)}, - filterName: GCEPDVolumeFilterType, - maxVols: 4, - fits: true, - test: "already-mounted EBS volumes are always ok to allow", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(GCEPDVolumeFilterType)}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "the same GCE volumes are not counted multiple times", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing two PVCs is counted towards the PV limit twice", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: deletedPVPod2, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing the same PV are counted towards the PV limit only once", - }, - { - newPod: anotherDeletedPVPod, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing different PVs are counted towards the PV limit twice", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: unboundPVCPod2, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", - }, - { - newPod: anotherUnboundPVCPod, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "two different unbound PVCs are counted towards the PV limit as two volumes", - }, - // filterName:AzureDiskVolumeFilterType - { - newPod: oneVolPod, - existingPods: []*v1.Pod{twoVolPod, oneVolPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 4, - fits: true, - test: "fits when node capacity >= new pod's AzureDisk volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "fit when node capacity < new pod's AzureDisk volumes", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{twoVolPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores non-AzureDisk volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts ignore non-AzureDisk volumes", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count considers PVCs backed by AzureDisk volumes", - }, - { - newPod: splitPVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores PVCs not backed by AzureDisk volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, onePVCPod(AzureDiskVolumeFilterType)}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts considers PVCs backed by AzureDisk volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(AzureDiskVolumeFilterType)}, - filterName: AzureDiskVolumeFilterType, - maxVols: 4, - fits: true, - test: "already-mounted AzureDisk volumes are always ok to allow", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(AzureDiskVolumeFilterType)}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "the same AzureDisk volumes are not counted multiple times", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing two PVCs is counted towards the PV limit twice", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: deletedPVPod2, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing the same PV are counted towards the PV limit only once", - }, - { - newPod: anotherDeletedPVPod, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing different PVs are counted towards the PV limit twice", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: unboundPVCPod2, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", - }, - { - newPod: anotherUnboundPVCPod, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "two different unbound PVCs are counted towards the PV limit as two volumes", - }, - } - - pvInfo := func(filterName string) FakePersistentVolumeInfo { - return FakePersistentVolumeInfo{ - { - ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"}, - Spec: v1.PersistentVolumeSpec{ - PersistentVolumeSource: v1.PersistentVolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: strings.ToLower(filterName) + "Vol"}, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"}, - Spec: v1.PersistentVolumeSpec{ - PersistentVolumeSource: v1.PersistentVolumeSource{}, - }, - }, - } - } - - pvcInfo := func(filterName string) FakePersistentVolumeClaimInfo { - return FakePersistentVolumeClaimInfo{ - { - ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: "some" + filterName + "Vol"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someNon" + filterName + "Vol"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "pvcWithDeletedPV"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: "pvcWithDeletedPV"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "anotherPVCWithDeletedPV"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: "anotherPVCWithDeletedPV"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "unboundPVC"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: ""}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "anotherUnboundPVC"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: ""}, - }, - } - } - - expectedFailureReasons := []algorithm.PredicateFailureReason{ErrMaxVolumeCountExceeded} - - for _, test := range tests { - os.Setenv(KubeMaxPDVols, strconv.Itoa(test.maxVols)) - pred := NewMaxPDVolumeCountPredicate(test.filterName, pvInfo(test.filterName), pvcInfo(test.filterName)) - fits, reasons, err := pred(test.newPod, PredicateMetadata(test.newPod, nil), schedulercache.NewNodeInfo(test.existingPods...)) - if err != nil { - t.Errorf("[%s]%s: unexpected error: %v", test.filterName, test.test, err) - } - if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { - t.Errorf("[%s]%s: unexpected failure reasons: %v, want: %v", test.filterName, test.test, reasons, expectedFailureReasons) - } - if fits != test.fits { - t.Errorf("[%s]%s: expected %v, got %v", test.filterName, test.test, test.fits, fits) - } - } -} - func newPodWithPort(hostPorts ...int) *v1.Pod { networkPorts := []v1.ContainerPort{} for _, port := range hostPorts { diff --git a/pkg/scheduler/cache/node_info.go b/pkg/scheduler/cache/node_info.go index 030938f6175..1479e81c554 100644 --- a/pkg/scheduler/cache/node_info.go +++ b/pkg/scheduler/cache/node_info.go @@ -412,6 +412,17 @@ func (n *NodeInfo) Clone() *NodeInfo { return clone } +// VolumeLimits returns volume limits associated with the node +func (n *NodeInfo) VolumeLimits() map[v1.ResourceName]int64 { + volumeLimits := map[v1.ResourceName]int64{} + for k, v := range n.AllocatableResource().ScalarResources { + if v1helper.IsAttachableVolumeResourceName(k) { + volumeLimits[k] = v + } + } + return volumeLimits +} + // String returns representation of human readable format of this NodeInfo. func (n *NodeInfo) String() string { podKeys := make([]string, len(n.pods)) diff --git a/pkg/scheduler/cache/node_info_test.go b/pkg/scheduler/cache/node_info_test.go index 1ab296d6b9a..7c9d9c039e9 100644 --- a/pkg/scheduler/cache/node_info_test.go +++ b/pkg/scheduler/cache/node_info_test.go @@ -84,7 +84,11 @@ func TestResourceList(t *testing.T) { Memory: 2000, EphemeralStorage: 5000, AllowedPodNumber: 80, - ScalarResources: map[v1.ResourceName]int64{"scalar.test/scalar1": 1, "hugepages-test": 2}, + ScalarResources: map[v1.ResourceName]int64{ + "scalar.test/scalar1": 1, + "hugepages-test": 2, + "attachable-volumes-aws-ebs": 39, + }, }, expected: map[v1.ResourceName]resource.Quantity{ v1.ResourceCPU: *resource.NewScaledQuantity(4, -3), @@ -92,6 +96,7 @@ func TestResourceList(t *testing.T) { v1.ResourcePods: *resource.NewQuantity(80, resource.BinarySI), v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), "scalar.test/" + "scalar1": *resource.NewQuantity(1, resource.DecimalSI), + "attachable-volumes-aws-ebs": *resource.NewQuantity(39, resource.DecimalSI), v1.ResourceHugePagesPrefix + "test": *resource.NewQuantity(2, resource.BinarySI), }, },