diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 693c4b57d1e..d612181288c 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -3602,6 +3602,8 @@ const ( ResourceDefaultNamespacePrefix = "kubernetes.io/" // Name prefix for huge page resources (alpha). ResourceHugePagesPrefix = "hugepages-" + // Name prefix for storage resource limits + ResourceAttachableVolumesPrefix = "attachable-volumes-" ) // ResourceList is a set of (resource name, quantity) pairs. diff --git a/pkg/apis/core/v1/helper/helpers.go b/pkg/apis/core/v1/helper/helpers.go index c7e2c2d0c7a..667ff54a20f 100644 --- a/pkg/apis/core/v1/helper/helpers.go +++ b/pkg/apis/core/v1/helper/helpers.go @@ -92,10 +92,14 @@ func IsOvercommitAllowed(name v1.ResourceName) bool { !IsHugePageResourceName(name) } +func IsAttachableVolumeResourceName(name v1.ResourceName) bool { + return strings.HasPrefix(string(name), v1.ResourceAttachableVolumesPrefix) +} + // Extended and Hugepages resources func IsScalarResourceName(name v1.ResourceName) bool { return IsExtendedResourceName(name) || IsHugePageResourceName(name) || - IsPrefixedNativeResource(name) + IsPrefixedNativeResource(name) || IsAttachableVolumeResourceName(name) } // this function aims to check if the service's ClusterIP is set or not diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 1cd0c51f0f6..2def762882a 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -5549,6 +5549,19 @@ func TestValidateContainers(t *testing.T) { TerminationMessagePolicy: "File", }, }, + "Invalid storage limit request": { + { + Name: "abc-123", + Image: "image", + Resources: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceName("attachable-volumes-aws-ebs"): *resource.NewQuantity(10, resource.DecimalSI), + }, + }, + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + }, + }, "Request limit multiple invalid": { { Name: "abc-123", diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 300c675653a..06290be4348 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -284,6 +284,13 @@ const ( // Do not remove this feature gate even though it's GA VolumeSubpath utilfeature.Feature = "VolumeSubpath" + // owner: @gnufied + // alpha : v1.11 + // + // Add support for volume plugins to report node specific + // volume limits + AttachVolumeLimit utilfeature.Feature = "AttachVolumeLimit" + // owner: @ravig // alpha: v1.11 // @@ -347,6 +354,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS QOSReserved: {Default: false, PreRelease: utilfeature.Alpha}, ExpandPersistentVolumes: {Default: true, PreRelease: utilfeature.Beta}, ExpandInUsePersistentVolumes: {Default: false, PreRelease: utilfeature.Alpha}, + AttachVolumeLimit: {Default: false, PreRelease: utilfeature.Alpha}, CPUManager: {Default: true, PreRelease: utilfeature.Beta}, ServiceNodeExclusion: {Default: false, PreRelease: utilfeature.Alpha}, MountContainers: {Default: false, PreRelease: utilfeature.Alpha}, diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index e63c5dc958f..ef4c3a5dafe 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -198,6 +198,9 @@ go_test( "//pkg/util/mount:go_default_library", "//pkg/version:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/aws_ebs:go_default_library", + "//pkg/volume/azure_dd:go_default_library", + "//pkg/volume/gce_pd:go_default_library", "//pkg/volume/host_path:go_default_library", "//pkg/volume/testing:go_default_library", "//pkg/volume/util:go_default_library", diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index 47400d3baf5..0eda88fbed8 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -326,6 +326,30 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) { return node, nil } +// setVolumeLimits updates volume limits on the node +func (kl *Kubelet) setVolumeLimits(node *v1.Node) { + if node.Status.Capacity == nil { + node.Status.Capacity = v1.ResourceList{} + } + + if node.Status.Allocatable == nil { + node.Status.Allocatable = v1.ResourceList{} + } + + pluginWithLimits := kl.volumePluginMgr.ListVolumePluginWithLimits() + for _, volumePlugin := range pluginWithLimits { + attachLimits, err := volumePlugin.GetVolumeLimits() + if err != nil { + glog.V(4).Infof("Error getting volume limit for plugin %s", volumePlugin.GetPluginName()) + continue + } + for limitKey, value := range attachLimits { + node.Status.Capacity[v1.ResourceName(limitKey)] = *resource.NewQuantity(value, resource.DecimalSI) + node.Status.Allocatable[v1.ResourceName(limitKey)] = *resource.NewQuantity(value, resource.DecimalSI) + } + } +} + // syncNodeStatus should be called periodically from a goroutine. // It synchronizes node status to master, registering the kubelet first if // necessary. @@ -751,6 +775,9 @@ func (kl *Kubelet) setNodeStatusInfo(node *v1.Node) { kl.setNodeStatusDaemonEndpoints(node) kl.setNodeStatusImages(node) kl.setNodeStatusGoRuntime(node) + if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { + kl.setVolumeLimits(node) + } } // Set Ready condition for the node. diff --git a/pkg/kubelet/kubelet_node_status_test.go b/pkg/kubelet/kubelet_node_status_test.go index 034560f14a3..d7115764284 100644 --- a/pkg/kubelet/kubelet_node_status_test.go +++ b/pkg/kubelet/kubelet_node_status_test.go @@ -331,7 +331,7 @@ func TestUpdateNewNodeStatus(t *testing.T) { } inputImageList, expectedImageList := generateTestingImageLists(numTestImages, int(tc.nodeStatusMaxImages)) testKubelet := newTestKubeletWithImageList( - t, inputImageList, false /* controllerAttachDetachEnabled */) + t, inputImageList, false /* controllerAttachDetachEnabled */, true /*initFakeVolumePlugin*/) defer testKubelet.Cleanup() kubelet := testKubelet.kubelet kubelet.nodeStatusMaxImages = tc.nodeStatusMaxImages @@ -1252,7 +1252,7 @@ func TestUpdateNewNodeStatusTooLargeReservation(t *testing.T) { // generate one more in inputImageList than we configure the Kubelet to report inputImageList, _ := generateTestingImageLists(nodeStatusMaxImages+1, nodeStatusMaxImages) testKubelet := newTestKubeletWithImageList( - t, inputImageList, false /* controllerAttachDetachEnabled */) + t, inputImageList, false /* controllerAttachDetachEnabled */, true /* initFakeVolumePlugin */) defer testKubelet.Cleanup() kubelet := testKubelet.kubelet kubelet.nodeStatusMaxImages = nodeStatusMaxImages @@ -1616,3 +1616,68 @@ func TestValidateNodeIPParam(t *testing.T) { } } } + +func TestSetVolumeLimits(t *testing.T) { + testKubelet := newTestKubeletWithoutFakeVolumePlugin(t, false /* controllerAttachDetachEnabled */) + defer testKubelet.Cleanup() + kubelet := testKubelet.kubelet + kubelet.kubeClient = nil // ensure only the heartbeat client is used + kubelet.hostname = testKubeletHostname + + var testcases = []struct { + name string + cloudProviderName string + expectedVolumeKey string + expectedLimit int64 + }{ + { + name: "For default GCE cloudprovider", + cloudProviderName: "gce", + expectedVolumeKey: util.GCEVolumeLimitKey, + expectedLimit: 16, + }, + { + name: "For default AWS Cloudprovider", + cloudProviderName: "aws", + expectedVolumeKey: util.EBSVolumeLimitKey, + expectedLimit: 39, + }, + { + name: "for default Azure cloudprovider", + cloudProviderName: "azure", + expectedVolumeKey: util.AzureVolumeLimitKey, + expectedLimit: 16, + }, + } + for _, test := range testcases { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname, Annotations: make(map[string]string)}, + Spec: v1.NodeSpec{}, + } + + fakeCloud := &fakecloud.FakeCloud{ + Provider: test.cloudProviderName, + Err: nil, + } + kubelet.cloud = fakeCloud + kubelet.cloudproviderRequestParallelism = make(chan int, 1) + kubelet.cloudproviderRequestSync = make(chan int) + kubelet.cloudproviderRequestTimeout = 10 * time.Second + kubelet.setVolumeLimits(node) + nodeLimits := []v1.ResourceList{} + nodeLimits = append(nodeLimits, node.Status.Allocatable) + nodeLimits = append(nodeLimits, node.Status.Capacity) + for _, volumeLimits := range nodeLimits { + fl, ok := volumeLimits[v1.ResourceName(test.expectedVolumeKey)] + if !ok { + t.Errorf("Expected to found volume limit for %s found none", test.expectedVolumeKey) + } + foundLimit, _ := fl.AsInt64() + expectedValue := resource.NewQuantity(test.expectedLimit, resource.DecimalSI) + if expectedValue.Cmp(fl) != 0 { + t.Errorf("Expected volume limit for %s to be %v found %v", test.expectedVolumeKey, test.expectedLimit, foundLimit) + } + } + + } +} diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 33b870c0b3c..3876f245152 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -67,6 +67,9 @@ import ( schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/aws_ebs" + "k8s.io/kubernetes/pkg/volume/azure_dd" + "k8s.io/kubernetes/pkg/volume/gce_pd" _ "k8s.io/kubernetes/pkg/volume/host_path" volumetest "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" @@ -133,13 +136,30 @@ func newTestKubelet(t *testing.T, controllerAttachDetachEnabled bool) *TestKubel Size: 456, }, } - return newTestKubeletWithImageList(t, imageList, controllerAttachDetachEnabled) + return newTestKubeletWithImageList(t, imageList, controllerAttachDetachEnabled, true /*initFakeVolumePlugin*/) +} + +func newTestKubeletWithoutFakeVolumePlugin(t *testing.T, controllerAttachDetachEnabled bool) *TestKubelet { + imageList := []kubecontainer.Image{ + { + ID: "abc", + RepoTags: []string{"k8s.gcr.io:v1", "k8s.gcr.io:v2"}, + Size: 123, + }, + { + ID: "efg", + RepoTags: []string{"k8s.gcr.io:v3", "k8s.gcr.io:v4"}, + Size: 456, + }, + } + return newTestKubeletWithImageList(t, imageList, controllerAttachDetachEnabled, false /*initFakeVolumePlugin*/) } func newTestKubeletWithImageList( t *testing.T, imageList []kubecontainer.Image, - controllerAttachDetachEnabled bool) *TestKubelet { + controllerAttachDetachEnabled bool, + initFakeVolumePlugin bool) *TestKubelet { fakeRuntime := &containertest.FakeRuntime{} fakeRuntime.RuntimeType = "test" fakeRuntime.VersionInfo = "1.5.0" @@ -293,10 +313,19 @@ func newTestKubeletWithImageList( // Add this as cleanup predicate pod admitter kubelet.admitHandlers.AddPodAdmitHandler(lifecycle.NewPredicateAdmitHandler(kubelet.getNodeAnyWay, lifecycle.NewAdmissionFailureHandlerStub(), kubelet.containerManager.UpdatePluginResources)) + allPlugins := []volume.VolumePlugin{} plug := &volumetest.FakeVolumePlugin{PluginName: "fake", Host: nil} + if initFakeVolumePlugin { + allPlugins = append(allPlugins, plug) + } else { + allPlugins = append(allPlugins, aws_ebs.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, gce_pd.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) + } + var prober volume.DynamicPluginProber = nil // TODO (#51147) inject mock kubelet.volumePluginMgr, err = - NewInitializedVolumePluginMgr(kubelet, kubelet.secretManager, kubelet.configMapManager, []volume.VolumePlugin{plug}, prober) + NewInitializedVolumePluginMgr(kubelet, kubelet.secretManager, kubelet.configMapManager, allPlugins, prober) require.NoError(t, err, "Failed to initialize VolumePluginMgr") kubelet.mounter = &mount.FakeMounter{} diff --git a/pkg/scheduler/algorithm/predicates/BUILD b/pkg/scheduler/algorithm/predicates/BUILD index bf8e46796d0..13e216174e6 100644 --- a/pkg/scheduler/algorithm/predicates/BUILD +++ b/pkg/scheduler/algorithm/predicates/BUILD @@ -46,6 +46,7 @@ go_library( go_test( name = "go_default_test", srcs = [ + "max_attachable_volume_predicate_test.go", "metadata_test.go", "predicates_test.go", "utils_test.go", @@ -53,10 +54,12 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core/v1/helper:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/cache:go_default_library", "//pkg/scheduler/testing:go_default_library", + "//pkg/volume/util:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", @@ -64,6 +67,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go new file mode 100644 index 00000000000..60aa5879a30 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go @@ -0,0 +1,854 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package predicates + +import ( + "os" + "reflect" + "strconv" + "strings" + "testing" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/scheduler/algorithm" + schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +func onePVCPod(filterName string) *v1.Pod { + return &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "some" + filterName + "Vol", + }, + }, + }, + }, + }, + } +} + +func splitPVCPod(filterName string) *v1.Pod { + return &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "someNon" + filterName + "Vol", + }, + }, + }, + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "some" + filterName + "Vol", + }, + }, + }, + }, + }, + } +} + +func TestVolumeCountConflicts(t *testing.T) { + oneVolPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp"}, + }, + }, + }, + }, + } + twoVolPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp1"}, + }, + }, + { + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp2"}, + }, + }, + }, + }, + } + splitVolsPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{}, + }, + }, + { + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "svp"}, + }, + }, + }, + }, + } + nonApplicablePod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{}, + }, + }, + }, + }, + } + deletedPVCPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "deletedPVC", + }, + }, + }, + }, + }, + } + twoDeletedPVCPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "deletedPVC", + }, + }, + }, + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "anotherDeletedPVC", + }, + }, + }, + }, + }, + } + deletedPVPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvcWithDeletedPV", + }, + }, + }, + }, + }, + } + // deletedPVPod2 is a different pod than deletedPVPod but using the same PVC + deletedPVPod2 := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvcWithDeletedPV", + }, + }, + }, + }, + }, + } + // anotherDeletedPVPod is a different pod than deletedPVPod and uses another PVC + anotherDeletedPVPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "anotherPVCWithDeletedPV", + }, + }, + }, + }, + }, + } + emptyPod := &v1.Pod{ + Spec: v1.PodSpec{}, + } + unboundPVCPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "unboundPVC", + }, + }, + }, + }, + }, + } + // Different pod than unboundPVCPod, but using the same unbound PVC + unboundPVCPod2 := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "unboundPVC", + }, + }, + }, + }, + }, + } + + // pod with unbound PVC that's different to unboundPVC + anotherUnboundPVCPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "anotherUnboundPVC", + }, + }, + }, + }, + }, + } + + tests := []struct { + newPod *v1.Pod + existingPods []*v1.Pod + filterName string + maxVols int + fits bool + test string + }{ + // filterName:EBSVolumeFilterType + { + newPod: oneVolPod, + existingPods: []*v1.Pod{twoVolPod, oneVolPod}, + filterName: EBSVolumeFilterType, + maxVols: 4, + fits: true, + test: "fits when node capacity >= new pod's EBS volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "doesn't fit when node capacity < new pod's EBS volumes", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{twoVolPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores non-EBS volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts ignore non-EBS volumes", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count considers PVCs backed by EBS volumes", + }, + { + newPod: splitPVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores PVCs not backed by EBS volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, onePVCPod(EBSVolumeFilterType)}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: false, + test: "existing pods' counts considers PVCs backed by EBS volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(EBSVolumeFilterType)}, + filterName: EBSVolumeFilterType, + maxVols: 4, + fits: true, + test: "already-mounted EBS volumes are always ok to allow", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(EBSVolumeFilterType)}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "the same EBS volumes are not counted multiple times", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: false, + test: "pod with missing two PVCs is counted towards the PV limit twice", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: deletedPVPod2, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing the same PV are counted towards the PV limit only once", + }, + { + newPod: anotherDeletedPVPod, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "two pods missing different PVs are counted towards the PV limit twice", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(EBSVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: unboundPVCPod2, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: true, + test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", + }, + { + newPod: anotherUnboundPVCPod, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: EBSVolumeFilterType, + maxVols: 2, + fits: false, + test: "two different unbound PVCs are counted towards the PV limit as two volumes", + }, + // filterName:GCEPDVolumeFilterType + { + newPod: oneVolPod, + existingPods: []*v1.Pod{twoVolPod, oneVolPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 4, + fits: true, + test: "fits when node capacity >= new pod's GCE volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "fit when node capacity < new pod's GCE volumes", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{twoVolPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores non-GCE volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts ignore non-GCE volumes", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count considers PVCs backed by GCE volumes", + }, + { + newPod: splitPVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores PVCs not backed by GCE volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, onePVCPod(GCEPDVolumeFilterType)}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts considers PVCs backed by GCE volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(GCEPDVolumeFilterType)}, + filterName: GCEPDVolumeFilterType, + maxVols: 4, + fits: true, + test: "already-mounted EBS volumes are always ok to allow", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(GCEPDVolumeFilterType)}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "the same GCE volumes are not counted multiple times", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing two PVCs is counted towards the PV limit twice", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: deletedPVPod2, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing the same PV are counted towards the PV limit only once", + }, + { + newPod: anotherDeletedPVPod, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing different PVs are counted towards the PV limit twice", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(GCEPDVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: unboundPVCPod2, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", + }, + { + newPod: anotherUnboundPVCPod, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: GCEPDVolumeFilterType, + maxVols: 2, + fits: true, + test: "two different unbound PVCs are counted towards the PV limit as two volumes", + }, + // filterName:AzureDiskVolumeFilterType + { + newPod: oneVolPod, + existingPods: []*v1.Pod{twoVolPod, oneVolPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 4, + fits: true, + test: "fits when node capacity >= new pod's AzureDisk volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "fit when node capacity < new pod's AzureDisk volumes", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{twoVolPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores non-AzureDisk volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts ignore non-AzureDisk volumes", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count considers PVCs backed by AzureDisk volumes", + }, + { + newPod: splitPVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "new pod's count ignores PVCs not backed by AzureDisk volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, onePVCPod(AzureDiskVolumeFilterType)}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "existing pods' counts considers PVCs backed by AzureDisk volumes", + }, + { + newPod: twoVolPod, + existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(AzureDiskVolumeFilterType)}, + filterName: AzureDiskVolumeFilterType, + maxVols: 4, + fits: true, + test: "already-mounted AzureDisk volumes are always ok to allow", + }, + { + newPod: splitVolsPod, + existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(AzureDiskVolumeFilterType)}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "the same AzureDisk volumes are not counted multiple times", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing two PVCs is counted towards the PV limit twice", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with missing PV is counted towards the PV limit", + }, + { + newPod: deletedPVPod2, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing the same PV are counted towards the PV limit only once", + }, + { + newPod: anotherDeletedPVPod, + existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "two pods missing different PVs are counted towards the PV limit twice", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: onePVCPod(AzureDiskVolumeFilterType), + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 3, + fits: true, + test: "pod with unbound PVC is counted towards the PV limit", + }, + { + newPod: unboundPVCPod2, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", + }, + { + newPod: anotherUnboundPVCPod, + existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, + filterName: AzureDiskVolumeFilterType, + maxVols: 2, + fits: true, + test: "two different unbound PVCs are counted towards the PV limit as two volumes", + }, + } + + pvInfo := func(filterName string) FakePersistentVolumeInfo { + return FakePersistentVolumeInfo{ + { + ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"}, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: strings.ToLower(filterName) + "Vol"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"}, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{}, + }, + }, + } + } + + pvcInfo := func(filterName string) FakePersistentVolumeClaimInfo { + return FakePersistentVolumeClaimInfo{ + { + ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "some" + filterName + "Vol"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someNon" + filterName + "Vol"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pvcWithDeletedPV"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "pvcWithDeletedPV"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "anotherPVCWithDeletedPV"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "anotherPVCWithDeletedPV"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "unboundPVC"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: ""}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "anotherUnboundPVC"}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: ""}, + }, + } + } + + expectedFailureReasons := []algorithm.PredicateFailureReason{ErrMaxVolumeCountExceeded} + + // running attachable predicate tests without feature gate and no limit present on nodes + for _, test := range tests { + os.Setenv(KubeMaxPDVols, strconv.Itoa(test.maxVols)) + pred := NewMaxPDVolumeCountPredicate(test.filterName, pvInfo(test.filterName), pvcInfo(test.filterName)) + fits, reasons, err := pred(test.newPod, PredicateMetadata(test.newPod, nil), schedulercache.NewNodeInfo(test.existingPods...)) + if err != nil { + t.Errorf("[%s]%s: unexpected error: %v", test.filterName, test.test, err) + } + if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { + t.Errorf("[%s]%s: unexpected failure reasons: %v, want: %v", test.filterName, test.test, reasons, expectedFailureReasons) + } + if fits != test.fits { + t.Errorf("[%s]%s: expected %v, got %v", test.filterName, test.test, test.fits, fits) + } + } + + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AttachVolumeLimit, true)() + + // running attachable predicate tests with feature gate and limit present on nodes + for _, test := range tests { + node := getNodeWithPodAndVolumeLimits(test.existingPods, int64(test.maxVols), test.filterName) + pred := NewMaxPDVolumeCountPredicate(test.filterName, pvInfo(test.filterName), pvcInfo(test.filterName)) + fits, reasons, err := pred(test.newPod, PredicateMetadata(test.newPod, nil), node) + if err != nil { + t.Errorf("Using allocatable [%s]%s: unexpected error: %v", test.filterName, test.test, err) + } + if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { + t.Errorf("Using allocatable [%s]%s: unexpected failure reasons: %v, want: %v", test.filterName, test.test, reasons, expectedFailureReasons) + } + if fits != test.fits { + t.Errorf("Using allocatable [%s]%s: expected %v, got %v", test.filterName, test.test, test.fits, fits) + } + } +} + +func getNodeWithPodAndVolumeLimits(pods []*v1.Pod, limit int64, filter string) *schedulercache.NodeInfo { + nodeInfo := schedulercache.NewNodeInfo(pods...) + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-for-max-pd-test-1"}, + Status: v1.NodeStatus{ + Allocatable: v1.ResourceList{ + getVolumeLimitKey(filter): *resource.NewQuantity(limit, resource.DecimalSI), + }, + }, + } + nodeInfo.SetNode(node) + return nodeInfo +} + +func getVolumeLimitKey(filterType string) v1.ResourceName { + switch filterType { + case EBSVolumeFilterType: + return v1.ResourceName(volumeutil.EBSVolumeLimitKey) + case GCEPDVolumeFilterType: + return v1.ResourceName(volumeutil.GCEVolumeLimitKey) + case AzureDiskVolumeFilterType: + return v1.ResourceName(volumeutil.AzureVolumeLimitKey) + default: + return "" + } +} diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 71a4688a4d6..aaf28f9622e 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -289,10 +289,11 @@ func NoDiskConflict(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *sch // MaxPDVolumeCountChecker contains information to check the max number of volumes for a predicate. type MaxPDVolumeCountChecker struct { - filter VolumeFilter - maxVolumes int - pvInfo PersistentVolumeInfo - pvcInfo PersistentVolumeClaimInfo + filter VolumeFilter + volumeLimitKey v1.ResourceName + maxVolumes int + pvInfo PersistentVolumeInfo + pvcInfo PersistentVolumeClaimInfo // The string below is generated randomly during the struct's initialization. // It is used to prefix volumeID generated inside the predicate() method to @@ -313,21 +314,25 @@ type VolumeFilter struct { // The predicate looks for both volumes used directly, as well as PVC volumes that are backed by relevant volume // types, counts the number of unique volumes, and rejects the new pod if it would place the total count over // the maximum. -func NewMaxPDVolumeCountPredicate(filterName string, pvInfo PersistentVolumeInfo, pvcInfo PersistentVolumeClaimInfo) algorithm.FitPredicate { - +func NewMaxPDVolumeCountPredicate( + filterName string, pvInfo PersistentVolumeInfo, pvcInfo PersistentVolumeClaimInfo) algorithm.FitPredicate { var filter VolumeFilter var maxVolumes int + var volumeLimitKey v1.ResourceName switch filterName { case EBSVolumeFilterType: filter = EBSVolumeFilter + volumeLimitKey = v1.ResourceName(volumeutil.EBSVolumeLimitKey) maxVolumes = getMaxVols(DefaultMaxEBSVolumes) case GCEPDVolumeFilterType: filter = GCEPDVolumeFilter + volumeLimitKey = v1.ResourceName(volumeutil.GCEVolumeLimitKey) maxVolumes = getMaxVols(DefaultMaxGCEPDVolumes) case AzureDiskVolumeFilterType: filter = AzureDiskVolumeFilter + volumeLimitKey = v1.ResourceName(volumeutil.AzureVolumeLimitKey) maxVolumes = getMaxVols(DefaultMaxAzureDiskVolumes) default: glog.Fatalf("Wrong filterName, Only Support %v %v %v ", EBSVolumeFilterType, @@ -337,6 +342,7 @@ func NewMaxPDVolumeCountPredicate(filterName string, pvInfo PersistentVolumeInfo } c := &MaxPDVolumeCountChecker{ filter: filter, + volumeLimitKey: volumeLimitKey, maxVolumes: maxVolumes, pvInfo: pvInfo, pvcInfo: pvcInfo, @@ -362,7 +368,6 @@ func getMaxVols(defaultVal int) int { } func (c *MaxPDVolumeCountChecker) filterVolumes(volumes []v1.Volume, namespace string, filteredVolumes map[string]bool) error { - for i := range volumes { vol := &volumes[i] if id, ok := c.filter.FilterVolume(vol); ok { @@ -449,15 +454,23 @@ func (c *MaxPDVolumeCountChecker) predicate(pod *v1.Pod, meta algorithm.Predicat } numNewVolumes := len(newVolumes) + maxAttachLimit := c.maxVolumes - if numExistingVolumes+numNewVolumes > c.maxVolumes { + if utilfeature.DefaultFeatureGate.Enabled(features.AttachVolumeLimit) { + volumeLimits := nodeInfo.VolumeLimits() + if maxAttachLimitFromAllocatable, ok := volumeLimits[c.volumeLimitKey]; ok { + maxAttachLimit = int(maxAttachLimitFromAllocatable) + } + } + + if numExistingVolumes+numNewVolumes > maxAttachLimit { // violates MaxEBSVolumeCount or MaxGCEPDVolumeCount return false, []algorithm.PredicateFailureReason{ErrMaxVolumeCountExceeded}, nil } if nodeInfo != nil && nodeInfo.TransientInfo != nil && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) { nodeInfo.TransientInfo.TransientLock.Lock() defer nodeInfo.TransientInfo.TransientLock.Unlock() - nodeInfo.TransientInfo.TransNodeInfo.AllocatableVolumesCount = c.maxVolumes - numExistingVolumes + nodeInfo.TransientInfo.TransNodeInfo.AllocatableVolumesCount = maxAttachLimit - numExistingVolumes nodeInfo.TransientInfo.TransNodeInfo.RequestedVolumes = numNewVolumes } return true, nil, nil diff --git a/pkg/scheduler/algorithm/predicates/predicates_test.go b/pkg/scheduler/algorithm/predicates/predicates_test.go index e6d2868d478..b664590f363 100644 --- a/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -1833,779 +1833,6 @@ func TestServiceAffinity(t *testing.T) { } } -func onePVCPod(filterName string) *v1.Pod { - return &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "some" + filterName + "Vol", - }, - }, - }, - }, - }, - } -} - -func splitPVCPod(filterName string) *v1.Pod { - return &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "someNon" + filterName + "Vol", - }, - }, - }, - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "some" + filterName + "Vol", - }, - }, - }, - }, - }, - } -} - -func TestVolumeCountConflicts(t *testing.T) { - oneVolPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp"}, - }, - }, - }, - }, - } - twoVolPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp1"}, - }, - }, - { - VolumeSource: v1.VolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp2"}, - }, - }, - }, - }, - } - splitVolsPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{}, - }, - }, - { - VolumeSource: v1.VolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "svp"}, - }, - }, - }, - }, - } - nonApplicablePod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{}, - }, - }, - }, - }, - } - deletedPVCPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "deletedPVC", - }, - }, - }, - }, - }, - } - twoDeletedPVCPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "deletedPVC", - }, - }, - }, - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "anotherDeletedPVC", - }, - }, - }, - }, - }, - } - deletedPVPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "pvcWithDeletedPV", - }, - }, - }, - }, - }, - } - // deletedPVPod2 is a different pod than deletedPVPod but using the same PVC - deletedPVPod2 := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "pvcWithDeletedPV", - }, - }, - }, - }, - }, - } - // anotherDeletedPVPod is a different pod than deletedPVPod and uses another PVC - anotherDeletedPVPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "anotherPVCWithDeletedPV", - }, - }, - }, - }, - }, - } - emptyPod := &v1.Pod{ - Spec: v1.PodSpec{}, - } - unboundPVCPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "unboundPVC", - }, - }, - }, - }, - }, - } - // Different pod than unboundPVCPod, but using the same unbound PVC - unboundPVCPod2 := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "unboundPVC", - }, - }, - }, - }, - }, - } - - // pod with unbound PVC that's different to unboundPVC - anotherUnboundPVCPod := &v1.Pod{ - Spec: v1.PodSpec{ - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "anotherUnboundPVC", - }, - }, - }, - }, - }, - } - - tests := []struct { - newPod *v1.Pod - existingPods []*v1.Pod - filterName string - maxVols int - fits bool - test string - }{ - // filterName:EBSVolumeFilterType - { - newPod: oneVolPod, - existingPods: []*v1.Pod{twoVolPod, oneVolPod}, - filterName: EBSVolumeFilterType, - maxVols: 4, - fits: true, - test: "fits when node capacity >= new pod's EBS volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "doesn't fit when node capacity < new pod's EBS volumes", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{twoVolPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores non-EBS volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts ignore non-EBS volumes", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count considers PVCs backed by EBS volumes", - }, - { - newPod: splitPVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores PVCs not backed by EBS volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, onePVCPod(EBSVolumeFilterType)}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: false, - test: "existing pods' counts considers PVCs backed by EBS volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(EBSVolumeFilterType)}, - filterName: EBSVolumeFilterType, - maxVols: 4, - fits: true, - test: "already-mounted EBS volumes are always ok to allow", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(EBSVolumeFilterType)}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "the same EBS volumes are not counted multiple times", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: false, - test: "pod with missing two PVCs is counted towards the PV limit twice", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: deletedPVPod2, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing the same PV are counted towards the PV limit only once", - }, - { - newPod: anotherDeletedPVPod, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "two pods missing different PVs are counted towards the PV limit twice", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(EBSVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: unboundPVCPod2, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: true, - test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", - }, - { - newPod: anotherUnboundPVCPod, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: EBSVolumeFilterType, - maxVols: 2, - fits: false, - test: "two different unbound PVCs are counted towards the PV limit as two volumes", - }, - // filterName:GCEPDVolumeFilterType - { - newPod: oneVolPod, - existingPods: []*v1.Pod{twoVolPod, oneVolPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 4, - fits: true, - test: "fits when node capacity >= new pod's GCE volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "fit when node capacity < new pod's GCE volumes", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{twoVolPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores non-GCE volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts ignore non-GCE volumes", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count considers PVCs backed by GCE volumes", - }, - { - newPod: splitPVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores PVCs not backed by GCE volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, onePVCPod(GCEPDVolumeFilterType)}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts considers PVCs backed by GCE volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(GCEPDVolumeFilterType)}, - filterName: GCEPDVolumeFilterType, - maxVols: 4, - fits: true, - test: "already-mounted EBS volumes are always ok to allow", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(GCEPDVolumeFilterType)}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "the same GCE volumes are not counted multiple times", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing two PVCs is counted towards the PV limit twice", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: deletedPVPod2, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing the same PV are counted towards the PV limit only once", - }, - { - newPod: anotherDeletedPVPod, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing different PVs are counted towards the PV limit twice", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(GCEPDVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: unboundPVCPod2, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", - }, - { - newPod: anotherUnboundPVCPod, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: GCEPDVolumeFilterType, - maxVols: 2, - fits: true, - test: "two different unbound PVCs are counted towards the PV limit as two volumes", - }, - // filterName:AzureDiskVolumeFilterType - { - newPod: oneVolPod, - existingPods: []*v1.Pod{twoVolPod, oneVolPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 4, - fits: true, - test: "fits when node capacity >= new pod's AzureDisk volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "fit when node capacity < new pod's AzureDisk volumes", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{twoVolPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores non-AzureDisk volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts ignore non-AzureDisk volumes", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count considers PVCs backed by AzureDisk volumes", - }, - { - newPod: splitPVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{splitVolsPod, oneVolPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "new pod's count ignores PVCs not backed by AzureDisk volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, onePVCPod(AzureDiskVolumeFilterType)}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "existing pods' counts considers PVCs backed by AzureDisk volumes", - }, - { - newPod: twoVolPod, - existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(AzureDiskVolumeFilterType)}, - filterName: AzureDiskVolumeFilterType, - maxVols: 4, - fits: true, - test: "already-mounted AzureDisk volumes are always ok to allow", - }, - { - newPod: splitVolsPod, - existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(AzureDiskVolumeFilterType)}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "the same AzureDisk volumes are not counted multiple times", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing two PVCs is counted towards the PV limit twice", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with missing PV is counted towards the PV limit", - }, - { - newPod: deletedPVPod2, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing the same PV are counted towards the PV limit only once", - }, - { - newPod: anotherDeletedPVPod, - existingPods: []*v1.Pod{oneVolPod, deletedPVPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "two pods missing different PVs are counted towards the PV limit twice", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: onePVCPod(AzureDiskVolumeFilterType), - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 3, - fits: true, - test: "pod with unbound PVC is counted towards the PV limit", - }, - { - newPod: unboundPVCPod2, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "the same unbound PVC in multiple pods is counted towards the PV limit only once", - }, - { - newPod: anotherUnboundPVCPod, - existingPods: []*v1.Pod{oneVolPod, unboundPVCPod}, - filterName: AzureDiskVolumeFilterType, - maxVols: 2, - fits: true, - test: "two different unbound PVCs are counted towards the PV limit as two volumes", - }, - } - - pvInfo := func(filterName string) FakePersistentVolumeInfo { - return FakePersistentVolumeInfo{ - { - ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"}, - Spec: v1.PersistentVolumeSpec{ - PersistentVolumeSource: v1.PersistentVolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: strings.ToLower(filterName) + "Vol"}, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"}, - Spec: v1.PersistentVolumeSpec{ - PersistentVolumeSource: v1.PersistentVolumeSource{}, - }, - }, - } - } - - pvcInfo := func(filterName string) FakePersistentVolumeClaimInfo { - return FakePersistentVolumeClaimInfo{ - { - ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: "some" + filterName + "Vol"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someNon" + filterName + "Vol"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "pvcWithDeletedPV"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: "pvcWithDeletedPV"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "anotherPVCWithDeletedPV"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: "anotherPVCWithDeletedPV"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "unboundPVC"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: ""}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "anotherUnboundPVC"}, - Spec: v1.PersistentVolumeClaimSpec{VolumeName: ""}, - }, - } - } - - expectedFailureReasons := []algorithm.PredicateFailureReason{ErrMaxVolumeCountExceeded} - - for _, test := range tests { - os.Setenv(KubeMaxPDVols, strconv.Itoa(test.maxVols)) - pred := NewMaxPDVolumeCountPredicate(test.filterName, pvInfo(test.filterName), pvcInfo(test.filterName)) - fits, reasons, err := pred(test.newPod, PredicateMetadata(test.newPod, nil), schedulercache.NewNodeInfo(test.existingPods...)) - if err != nil { - t.Errorf("[%s]%s: unexpected error: %v", test.filterName, test.test, err) - } - if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { - t.Errorf("[%s]%s: unexpected failure reasons: %v, want: %v", test.filterName, test.test, reasons, expectedFailureReasons) - } - if fits != test.fits { - t.Errorf("[%s]%s: expected %v, got %v", test.filterName, test.test, test.fits, fits) - } - } -} - func newPodWithPort(hostPorts ...int) *v1.Pod { networkPorts := []v1.ContainerPort{} for _, port := range hostPorts { diff --git a/pkg/scheduler/cache/node_info.go b/pkg/scheduler/cache/node_info.go index 030938f6175..1479e81c554 100644 --- a/pkg/scheduler/cache/node_info.go +++ b/pkg/scheduler/cache/node_info.go @@ -412,6 +412,17 @@ func (n *NodeInfo) Clone() *NodeInfo { return clone } +// VolumeLimits returns volume limits associated with the node +func (n *NodeInfo) VolumeLimits() map[v1.ResourceName]int64 { + volumeLimits := map[v1.ResourceName]int64{} + for k, v := range n.AllocatableResource().ScalarResources { + if v1helper.IsAttachableVolumeResourceName(k) { + volumeLimits[k] = v + } + } + return volumeLimits +} + // String returns representation of human readable format of this NodeInfo. func (n *NodeInfo) String() string { podKeys := make([]string, len(n.pods)) diff --git a/pkg/scheduler/cache/node_info_test.go b/pkg/scheduler/cache/node_info_test.go index 1ab296d6b9a..7c9d9c039e9 100644 --- a/pkg/scheduler/cache/node_info_test.go +++ b/pkg/scheduler/cache/node_info_test.go @@ -84,7 +84,11 @@ func TestResourceList(t *testing.T) { Memory: 2000, EphemeralStorage: 5000, AllowedPodNumber: 80, - ScalarResources: map[v1.ResourceName]int64{"scalar.test/scalar1": 1, "hugepages-test": 2}, + ScalarResources: map[v1.ResourceName]int64{ + "scalar.test/scalar1": 1, + "hugepages-test": 2, + "attachable-volumes-aws-ebs": 39, + }, }, expected: map[v1.ResourceName]resource.Quantity{ v1.ResourceCPU: *resource.NewScaledQuantity(4, -3), @@ -92,6 +96,7 @@ func TestResourceList(t *testing.T) { v1.ResourcePods: *resource.NewQuantity(80, resource.BinarySI), v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), "scalar.test/" + "scalar1": *resource.NewQuantity(1, resource.DecimalSI), + "attachable-volumes-aws-ebs": *resource.NewQuantity(39, resource.DecimalSI), v1.ResourceHugePagesPrefix + "test": *resource.NewQuantity(2, resource.BinarySI), }, }, diff --git a/pkg/volume/aws_ebs/aws_ebs.go b/pkg/volume/aws_ebs/aws_ebs.go index 2dedb9aa4af..d91ace3041c 100644 --- a/pkg/volume/aws_ebs/aws_ebs.go +++ b/pkg/volume/aws_ebs/aws_ebs.go @@ -17,9 +17,11 @@ limitations under the License. package aws_ebs import ( + "context" "fmt" "os" "path/filepath" + "regexp" "strconv" "strings" @@ -95,6 +97,39 @@ func (plugin *awsElasticBlockStorePlugin) SupportsBulkVolumeVerification() bool return true } +func (plugin *awsElasticBlockStorePlugin) GetVolumeLimits() (map[string]int64, error) { + cloud := plugin.host.GetCloudProvider() + + if cloud.ProviderName() != aws.ProviderName { + return nil, fmt.Errorf("Expected aws cloud, found %s", cloud.ProviderName()) + } + + volumeLimits := map[string]int64{ + util.EBSVolumeLimitKey: 39, + } + instances, ok := cloud.Instances() + if !ok { + glog.V(3).Infof("Failed to get instances from cloud provider") + return volumeLimits, nil + } + + instanceType, err := instances.InstanceType(context.TODO(), plugin.host.GetNodeName()) + if err != nil { + glog.Errorf("Failed to get instance type from AWS cloud provider") + return volumeLimits, nil + } + + if ok, _ := regexp.MatchString("^[cm]5.*", instanceType); ok { + volumeLimits[util.EBSVolumeLimitKey] = 25 + } + + return volumeLimits, nil +} + +func (plugin *awsElasticBlockStorePlugin) VolumeLimitKey(spec *volume.Spec) string { + return util.EBSVolumeLimitKey +} + func (plugin *awsElasticBlockStorePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, @@ -267,6 +302,7 @@ func (plugin *awsElasticBlockStorePlugin) ExpandVolumeDevice( } var _ volume.ExpandableVolumePlugin = &awsElasticBlockStorePlugin{} +var _ volume.VolumePluginWithAttachLimits = &awsElasticBlockStorePlugin{} // Abstract interface to PD operations. type ebsManager interface { diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index ec7ab8debf5..11aba03a133 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -17,13 +17,17 @@ limitations under the License. package azure_dd import ( + "fmt" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage" "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" ) // interface exposed by the cloud provider implementing Disk functionality @@ -62,6 +66,7 @@ var _ volume.PersistentVolumePlugin = &azureDataDiskPlugin{} var _ volume.DeletableVolumePlugin = &azureDataDiskPlugin{} var _ volume.ProvisionableVolumePlugin = &azureDataDiskPlugin{} var _ volume.AttachableVolumePlugin = &azureDataDiskPlugin{} +var _ volume.VolumePluginWithAttachLimits = &azureDataDiskPlugin{} const ( azureDataDiskPluginName = "kubernetes.io/azure-disk" @@ -106,6 +111,22 @@ func (plugin *azureDataDiskPlugin) SupportsBulkVolumeVerification() bool { return false } +func (plugin *azureDataDiskPlugin) GetVolumeLimits() (map[string]int64, error) { + cloud := plugin.host.GetCloudProvider() + if cloud.ProviderName() != azure.CloudProviderName { + return nil, fmt.Errorf("Expected Azure cloudprovider, got %s", cloud.ProviderName()) + } + + volumeLimits := map[string]int64{ + util.AzureVolumeLimitKey: 16, + } + return volumeLimits, nil +} + +func (plugin *azureDataDiskPlugin) VolumeLimitKey(spec *volume.Spec) string { + return util.AzureVolumeLimitKey +} + func (plugin *azureDataDiskPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/gce_pd/gce_pd.go b/pkg/volume/gce_pd/gce_pd.go index 9c2be1c8133..01b34628d67 100644 --- a/pkg/volume/gce_pd/gce_pd.go +++ b/pkg/volume/gce_pd/gce_pd.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" + gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" kstrings "k8s.io/kubernetes/pkg/util/strings" @@ -49,6 +50,7 @@ var _ volume.PersistentVolumePlugin = &gcePersistentDiskPlugin{} var _ volume.DeletableVolumePlugin = &gcePersistentDiskPlugin{} var _ volume.ProvisionableVolumePlugin = &gcePersistentDiskPlugin{} var _ volume.ExpandableVolumePlugin = &gcePersistentDiskPlugin{} +var _ volume.VolumePluginWithAttachLimits = &gcePersistentDiskPlugin{} const ( gcePersistentDiskPluginName = "kubernetes.io/gce-pd" @@ -100,6 +102,23 @@ func (plugin *gcePersistentDiskPlugin) GetAccessModes() []v1.PersistentVolumeAcc } } +func (plugin *gcePersistentDiskPlugin) GetVolumeLimits() (map[string]int64, error) { + cloud := plugin.host.GetCloudProvider() + + if cloud.ProviderName() != gcecloud.ProviderName { + return nil, fmt.Errorf("Expected gce cloud got %s", cloud.ProviderName()) + } + + volumeLimits := map[string]int64{ + util.GCEVolumeLimitKey: 16, + } + return volumeLimits, nil +} + +func (plugin *gcePersistentDiskPlugin) VolumeLimitKey(spec *volume.Spec) string { + return util.GCEVolumeLimitKey +} + func (plugin *gcePersistentDiskPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { // Inject real implementations here, test through the internal function. return plugin.newMounterInternal(spec, pod.UID, &GCEDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 5ed32f4be92..b95ad471ca9 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -216,6 +216,32 @@ type ExpandableVolumePlugin interface { RequiresFSResize() bool } +// VolumePluginWithAttachLimits is an extended interface of VolumePlugin that restricts number of +// volumes that can be attached to a node. +type VolumePluginWithAttachLimits interface { + VolumePlugin + // Return maximum number of volumes that can be attached to a node for this plugin. + // The key must be same as string returned by VolumeLimitKey function. The returned + // map may look like: + // - { "storage-limits-aws-ebs": 39 } + // - { "storage-limits-gce-pd": 10 } + // A volume plugin may return error from this function - if it can not be used on a given node or not + // applicable in given environment (where environment could be cloudprovider or any other dependency) + // For example - calling this function for EBS volume plugin on a GCE node should + // result in error. + // The returned values are stored in node allocatable property and will be used + // by scheduler to determine how many pods with volumes can be scheduled on given node. + GetVolumeLimits() (map[string]int64, error) + // Return volume limit key string to be used in node capacity constraints + // The key must start with prefix storage-limits-. For example: + // - storage-limits-aws-ebs + // - storage-limits-csi-cinder + // The key should respect character limit of ResourceName type + // This function may be called by kubelet or scheduler to identify node allocatable property + // which stores volumes limits. + VolumeLimitKey(spec *Spec) string +} + // BlockVolumePlugin is an extend interface of VolumePlugin and is used for block volumes support. type BlockVolumePlugin interface { VolumePlugin @@ -584,6 +610,17 @@ func (pm *VolumePluginMgr) refreshProbedPlugins() { } } +// ListVolumePluginWithLimits returns plugins that have volume limits on nodes +func (pm *VolumePluginMgr) ListVolumePluginWithLimits() []VolumePluginWithAttachLimits { + matchedPlugins := []VolumePluginWithAttachLimits{} + for _, v := range pm.plugins { + if plugin, ok := v.(VolumePluginWithAttachLimits); ok { + matchedPlugins = append(matchedPlugins, plugin) + } + } + return matchedPlugins +} + // FindPersistentPluginBySpec looks for a persistent volume plugin that can // support a given volume specification. If no plugin is found, return an // error @@ -598,6 +635,20 @@ func (pm *VolumePluginMgr) FindPersistentPluginBySpec(spec *Spec) (PersistentVol return nil, fmt.Errorf("no persistent volume plugin matched") } +// FindVolumePluginWithLimitsBySpec returns volume plugin that has a limit on how many +// of them can be attached to a node +func (pm *VolumePluginMgr) FindVolumePluginWithLimitsBySpec(spec *Spec) (VolumePluginWithAttachLimits, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, fmt.Errorf("Could not find volume plugin for spec : %#v", spec) + } + + if limitedPlugin, ok := volumePlugin.(VolumePluginWithAttachLimits); ok { + return limitedPlugin, nil + } + return nil, fmt.Errorf("no plugin with limits found") +} + // FindPersistentPluginByName fetches a persistent volume plugin by name. If // no plugin is found, returns error. func (pm *VolumePluginMgr) FindPersistentPluginByName(name string) (PersistentVolumePlugin, error) { diff --git a/pkg/volume/util/BUILD b/pkg/volume/util/BUILD index 1002d7eeecf..1a74224fb81 100644 --- a/pkg/volume/util/BUILD +++ b/pkg/volume/util/BUILD @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "atomic_writer.go", + "attach_limit.go", "device_util.go", "doc.go", "error.go", diff --git a/pkg/volume/util/attach_limit.go b/pkg/volume/util/attach_limit.go new file mode 100644 index 00000000000..610f5f5b2cc --- /dev/null +++ b/pkg/volume/util/attach_limit.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +// This file is a common place holder for volume limit utility constants +// shared between volume package and scheduler + +const ( + // EBSVolumeLimitKey resource name that will store volume limits for EBS + EBSVolumeLimitKey = "attachable-volumes-aws-ebs" + // AzureVolumeLimitKey stores resource name that will store volume limits for Azure + AzureVolumeLimitKey = "attachable-volumes-azure-disk" + // GCEVolumeLimitKey stores resource name that will store volume limits for GCE node + GCEVolumeLimitKey = "attachable-volumes-gce-pd" +) diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index cb5b2edd230..5fb4a12fb59 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -4040,6 +4040,8 @@ const ( ResourceDefaultNamespacePrefix = "kubernetes.io/" // Name prefix for huge page resources (alpha). ResourceHugePagesPrefix = "hugepages-" + // Name prefix for storage resource limits + ResourceAttachableVolumesPrefix = "attachable-volumes-" ) // ResourceList is a set of (resource name, quantity) pairs.