mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-21 10:51:29 +00:00
Merge pull request #67121 from feiskyer/azdisk-affinity
Automatic merge from submit-queue (batch tested with PRs 67061, 66589, 67121, 67149). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add DynamicProvisioningScheduling and VolumeScheduling support for Azure managed disks **What this PR does / why we need it**: Continue of [Azure Availability Zone feature](https://github.com/kubernetes/features/issues/586). This PR adds `VolumeScheduling` and `DynamicProvisioningScheduling` support to Azure managed disks. When feature gate `VolumeScheduling` disabled, no NodeAffinity set for PV: ```yaml kubectl describe pv Name: pvc-d30dad05-9ad8-11e8-94f2-000d3a07de8c Labels: failure-domain.beta.kubernetes.io/region=southeastasia failure-domain.beta.kubernetes.io/zone=southeastasia-2 Annotations: pv.kubernetes.io/bound-by-controller=yes pv.kubernetes.io/provisioned-by=kubernetes.io/azure-disk volumehelper.VolumeDynamicallyCreatedByKey=azure-disk-dynamic-provisioner Finalizers: [kubernetes.io/pv-protection] StorageClass: default Status: Bound Claim: default/pvc-azuredisk Reclaim Policy: Delete Access Modes: RWO Capacity: 5Gi Node Affinity: Required Terms: Term 0: failure-domain.beta.kubernetes.io/region in [southeastasia] failure-domain.beta.kubernetes.io/zone in [southeastasia-2] Message: Source: Type: AzureDisk (an Azure Data Disk mount on the host and bind mount to the pod) DiskName: k8s-5b3d7b8f-dynamic-pvc-d30dad05-9ad8-11e8-94f2-000d3a07de8c DiskURI: /subscriptions/<subscription-id>/resourceGroups/<rg-name>/providers/Microsoft.Compute/disks/k8s-5b3d7b8f-dynamic-pvc-d30dad05-9ad8-11e8-94f2-000d3a07de8c Kind: Managed FSType: CachingMode: None ReadOnly: false Events: <none> ``` When feature gate `VolumeScheduling` enabled, NodeAffinity will be populated for PV: ```yaml kubectl describe pv Name: pvc-0284337b-9ada-11e8-a7f6-000d3a07de8c Labels: failure-domain.beta.kubernetes.io/region=southeastasia failure-domain.beta.kubernetes.io/zone=southeastasia-2 Annotations: pv.kubernetes.io/bound-by-controller=yes pv.kubernetes.io/provisioned-by=kubernetes.io/azure-disk volumehelper.VolumeDynamicallyCreatedByKey=azure-disk-dynamic-provisioner Finalizers: [kubernetes.io/pv-protection] StorageClass: default Status: Bound Claim: default/pvc-azuredisk Reclaim Policy: Delete Access Modes: RWO Capacity: 5Gi Node Affinity: Required Terms: Term 0: failure-domain.beta.kubernetes.io/region in [southeastasia] failure-domain.beta.kubernetes.io/zone in [southeastasia-2] Message: Source: Type: AzureDisk (an Azure Data Disk mount on the host and bind mount to the pod) DiskName: k8s-5b3d7b8f-dynamic-pvc-0284337b-9ada-11e8-a7f6-000d3a07de8c DiskURI: /subscriptions/<subscription-id>/resourceGroups/<rg-name>/providers/Microsoft.Compute/disks/k8s-5b3d7b8f-dynamic-pvc-0284337b-9ada-11e8-a7f6-000d3a07de8c Kind: Managed FSType: CachingMode: None ReadOnly: false Events: <none> ``` When both `VolumeScheduling` and `DynamicProvisioningScheduling` are enabled, storage class also supports `allowedTopologies` and `volumeBindingMode: WaitForFirstConsumer` for volume topology aware dynamic provisioning: ```yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: annotations: name: managed-disk-dynamic parameters: cachingmode: None kind: Managed storageaccounttype: Standard_LRS provisioner: kubernetes.io/azure-disk reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer allowedTopologies: - matchLabelExpressions: - key: failure-domain.beta.kubernetes.io/zone values: - southeastasia-2 - southeastasia-1 ``` **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: **Release note**: ```release-note DynamicProvisioningScheduling and VolumeScheduling is not supported for Azure managed disks. Feature gates DynamicProvisioningScheduling and VolumeScheduling should be enabled before using this feature. ``` /kind feature /sig azure /cc @brendandburns @khenidak @andyzhangx /cc @ddebroy @msau42 @justaugustus
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
ae351f1184
@ -29,7 +29,6 @@ import (
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
kwait "k8s.io/apimachinery/pkg/util/wait"
|
||||
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
|
||||
"k8s.io/kubernetes/pkg/volume"
|
||||
@ -51,16 +50,8 @@ type ManagedDiskOptions struct {
|
||||
PVCName string
|
||||
// The name of resource group.
|
||||
ResourceGroup string
|
||||
// Wether the disk is zoned.
|
||||
Zoned bool
|
||||
// Wether AvailabilityZone is set.
|
||||
ZonePresent bool
|
||||
// Wether AvailabilityZones is set.
|
||||
ZonesPresent bool
|
||||
// The AvailabilityZone to create the disk.
|
||||
AvailabilityZone string
|
||||
// List of AvailabilityZone to create the disk.
|
||||
AvailabilityZones string
|
||||
// The tags of the disk.
|
||||
Tags map[string]string
|
||||
// The SKU of storage account.
|
||||
@ -73,44 +64,12 @@ func newManagedDiskController(common *controllerCommon) (*ManagedDiskController,
|
||||
|
||||
//CreateManagedDisk : create managed disk
|
||||
func (c *ManagedDiskController) CreateManagedDisk(options *ManagedDiskOptions) (string, error) {
|
||||
var zones sets.String
|
||||
var activeZones sets.String
|
||||
var err error
|
||||
glog.V(4).Infof("azureDisk - creating new managed Name:%s StorageAccountType:%s Size:%v", options.DiskName, options.StorageAccountType, options.SizeGB)
|
||||
|
||||
// Get active zones which have nodes running on.
|
||||
activeZones, err = c.common.cloud.GetActiveZones()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error querying active zones: %v", err)
|
||||
}
|
||||
|
||||
// Validate and choose availability zone for creating disk.
|
||||
if options.Zoned && !options.ZonePresent && !options.ZonesPresent {
|
||||
// Neither "zone" or "zones" specified. Pick a zone randomly selected
|
||||
// from all active zones where Kubernetes cluster has a node.
|
||||
zones = activeZones
|
||||
} else if !options.ZonePresent && options.ZonesPresent {
|
||||
// Choose zone from specified zones.
|
||||
if zones, err = util.ZonesToSet(options.AvailabilityZones); err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else if options.ZonePresent && !options.ZonesPresent {
|
||||
if err := util.ValidateZone(options.AvailabilityZone); err != nil {
|
||||
return "", err
|
||||
}
|
||||
zones = make(sets.String)
|
||||
zones.Insert(options.AvailabilityZone)
|
||||
}
|
||||
var createZones *[]string
|
||||
if len(zones.List()) > 0 {
|
||||
createAZ := util.ChooseZoneForVolume(zones, options.PVCName)
|
||||
// Do not allow creation of disks in zones that are do not have nodes. Such disks
|
||||
// are not currently usable.
|
||||
if !activeZones.Has(createAZ) {
|
||||
return "", fmt.Errorf("kubernetes does not have a node in zone %q", createAZ)
|
||||
}
|
||||
|
||||
zoneList := []string{c.common.cloud.GetZoneID(createAZ)}
|
||||
if len(options.AvailabilityZone) > 0 {
|
||||
zoneList := []string{c.common.cloud.GetZoneID(options.AvailabilityZone)}
|
||||
createZones = &zoneList
|
||||
}
|
||||
|
||||
|
||||
@ -22,9 +22,11 @@ import (
|
||||
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute"
|
||||
"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage"
|
||||
"github.com/golang/glog"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
|
||||
"k8s.io/kubernetes/pkg/volume"
|
||||
"k8s.io/kubernetes/pkg/volume/util"
|
||||
@ -61,6 +63,9 @@ type DiskController interface {
|
||||
|
||||
// GetAzureDiskLabels gets availability zone labels for Azuredisk.
|
||||
GetAzureDiskLabels(diskURI string) (map[string]string, error)
|
||||
|
||||
// GetActiveZones returns all the zones in which k8s nodes are currently running.
|
||||
GetActiveZones() (sets.String, error)
|
||||
}
|
||||
|
||||
type azureDataDiskPlugin struct {
|
||||
|
||||
@ -25,6 +25,7 @@ import (
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
@ -118,12 +119,13 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
|
||||
err error
|
||||
resourceGroup string
|
||||
|
||||
zoned bool
|
||||
zonePresent bool
|
||||
zonesPresent bool
|
||||
strZoned string
|
||||
availabilityZone string
|
||||
availabilityZones string
|
||||
zoned bool
|
||||
zonePresent bool
|
||||
zonesPresent bool
|
||||
strZoned string
|
||||
availabilityZone string
|
||||
availabilityZones sets.String
|
||||
selectedAvailabilityZone string
|
||||
)
|
||||
// maxLength = 79 - (4 for ".vhd") = 75
|
||||
name := util.GenerateVolumeName(p.options.ClusterName, p.options.PVName, 75)
|
||||
@ -156,7 +158,10 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
|
||||
availabilityZone = v
|
||||
case "zones":
|
||||
zonesPresent = true
|
||||
availabilityZones = v
|
||||
availabilityZones, err = util.ZonesToSet(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing zones %s, must be strings separated by commas: %v", v, err)
|
||||
}
|
||||
case "zoned":
|
||||
strZoned = v
|
||||
default:
|
||||
@ -175,6 +180,16 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if kind != v1.AzureManagedDisk {
|
||||
if resourceGroup != "" {
|
||||
return nil, errors.New("StorageClass option 'resourceGroup' can be used only for managed disks")
|
||||
}
|
||||
|
||||
if zoned {
|
||||
return nil, errors.New("StorageClass option 'zoned' parameter is only supported for managed disks")
|
||||
}
|
||||
}
|
||||
|
||||
zoned, err = parseZoned(strZoned, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -184,10 +199,6 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
|
||||
return nil, fmt.Errorf("zone or zones StorageClass parameters must be used together with zoned parameter")
|
||||
}
|
||||
|
||||
if zonePresent && zonesPresent {
|
||||
return nil, fmt.Errorf("both zone and zones StorageClass parameters must not be used at the same time")
|
||||
}
|
||||
|
||||
if cachingMode, err = normalizeCachingMode(cachingMode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -197,8 +208,17 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resourceGroup != "" && kind != v1.AzureManagedDisk {
|
||||
return nil, errors.New("StorageClass option 'resourceGroup' can be used only for managed disks")
|
||||
// Select zone for managed disks based on zone, zones and allowedTopologies.
|
||||
if zoned {
|
||||
activeZones, err := diskController.GetActiveZones()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error querying active zones: %v", err)
|
||||
}
|
||||
|
||||
selectedAvailabilityZone, err = util.SelectZoneForVolume(zonePresent, zonesPresent, availabilityZone, availabilityZones, activeZones, selectedNode, allowedTopologies, p.options.PVC.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// create disk
|
||||
@ -217,11 +237,7 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
|
||||
PVCName: p.options.PVC.Name,
|
||||
SizeGB: requestGiB,
|
||||
Tags: tags,
|
||||
Zoned: zoned,
|
||||
ZonePresent: zonePresent,
|
||||
ZonesPresent: zonesPresent,
|
||||
AvailabilityZone: availabilityZone,
|
||||
AvailabilityZones: availabilityZones,
|
||||
AvailabilityZone: selectedAvailabilityZone,
|
||||
}
|
||||
diskURI, err = diskController.CreateManagedDisk(volumeOptions)
|
||||
if err != nil {
|
||||
@ -232,10 +248,6 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if zoned {
|
||||
return nil, errors.New("zoned parameter is only supported for managed disks")
|
||||
}
|
||||
|
||||
if kind == v1.AzureDedicatedBlobDisk {
|
||||
_, diskURI, _, err = diskController.CreateVolume(name, account, storageAccountType, location, requestGiB)
|
||||
if err != nil {
|
||||
@ -286,5 +298,21 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
|
||||
},
|
||||
}
|
||||
|
||||
if zoned && utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
|
||||
requirements := make([]v1.NodeSelectorRequirement, 0)
|
||||
for k, v := range labels {
|
||||
requirements = append(requirements, v1.NodeSelectorRequirement{Key: k, Operator: v1.NodeSelectorOpIn, Values: []string{v}})
|
||||
}
|
||||
|
||||
nodeSelectorTerm := v1.NodeSelectorTerm{
|
||||
MatchExpressions: requirements,
|
||||
}
|
||||
pv.Spec.NodeAffinity = &v1.VolumeNodeAffinity{
|
||||
Required: &v1.NodeSelector{
|
||||
NodeSelectorTerms: []v1.NodeSelectorTerm{nodeSelectorTerm},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return pv, nil
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user