Merge pull request #67121 from feiskyer/azdisk-affinity

Automatic merge from submit-queue (batch tested with PRs 67061, 66589, 67121, 67149). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add DynamicProvisioningScheduling and VolumeScheduling support for Azure managed disks

**What this PR does / why we need it**:

Continue of [Azure Availability Zone feature](https://github.com/kubernetes/features/issues/586).

This PR adds `VolumeScheduling` and `DynamicProvisioningScheduling` support to Azure managed disks.

When feature gate `VolumeScheduling` disabled, no NodeAffinity set for PV:

```yaml
kubectl describe pv
Name:              pvc-d30dad05-9ad8-11e8-94f2-000d3a07de8c
Labels:            failure-domain.beta.kubernetes.io/region=southeastasia
                   failure-domain.beta.kubernetes.io/zone=southeastasia-2
Annotations:       pv.kubernetes.io/bound-by-controller=yes
                   pv.kubernetes.io/provisioned-by=kubernetes.io/azure-disk
                   volumehelper.VolumeDynamicallyCreatedByKey=azure-disk-dynamic-provisioner
Finalizers:        [kubernetes.io/pv-protection]
StorageClass:      default
Status:            Bound
Claim:             default/pvc-azuredisk
Reclaim Policy:    Delete
Access Modes:      RWO
Capacity:          5Gi
Node Affinity:
  Required Terms:
    Term 0:        failure-domain.beta.kubernetes.io/region in [southeastasia]
                   failure-domain.beta.kubernetes.io/zone in [southeastasia-2]
Message:
Source:
    Type:         AzureDisk (an Azure Data Disk mount on the host and bind mount to the pod)
    DiskName:     k8s-5b3d7b8f-dynamic-pvc-d30dad05-9ad8-11e8-94f2-000d3a07de8c
    DiskURI:      /subscriptions/<subscription-id>/resourceGroups/<rg-name>/providers/Microsoft.Compute/disks/k8s-5b3d7b8f-dynamic-pvc-d30dad05-9ad8-11e8-94f2-000d3a07de8c
    Kind:         Managed
    FSType:
    CachingMode:  None
    ReadOnly:     false
Events:           <none>
```

When feature gate `VolumeScheduling` enabled, NodeAffinity will be populated for PV:

```yaml
kubectl describe pv
Name:              pvc-0284337b-9ada-11e8-a7f6-000d3a07de8c
Labels:            failure-domain.beta.kubernetes.io/region=southeastasia
                   failure-domain.beta.kubernetes.io/zone=southeastasia-2
Annotations:       pv.kubernetes.io/bound-by-controller=yes
                   pv.kubernetes.io/provisioned-by=kubernetes.io/azure-disk
                   volumehelper.VolumeDynamicallyCreatedByKey=azure-disk-dynamic-provisioner
Finalizers:        [kubernetes.io/pv-protection]
StorageClass:      default
Status:            Bound
Claim:             default/pvc-azuredisk
Reclaim Policy:    Delete
Access Modes:      RWO
Capacity:          5Gi
Node Affinity:
  Required Terms:
    Term 0:        failure-domain.beta.kubernetes.io/region in [southeastasia]
                   failure-domain.beta.kubernetes.io/zone in [southeastasia-2]
Message:
Source:
    Type:         AzureDisk (an Azure Data Disk mount on the host and bind mount to the pod)
    DiskName:     k8s-5b3d7b8f-dynamic-pvc-0284337b-9ada-11e8-a7f6-000d3a07de8c
    DiskURI:      /subscriptions/<subscription-id>/resourceGroups/<rg-name>/providers/Microsoft.Compute/disks/k8s-5b3d7b8f-dynamic-pvc-0284337b-9ada-11e8-a7f6-000d3a07de8c
    Kind:         Managed
    FSType:
    CachingMode:  None
    ReadOnly:     false
Events:           <none>
```

When both  `VolumeScheduling` and `DynamicProvisioningScheduling` are enabled, storage class also supports `allowedTopologies` and `volumeBindingMode: WaitForFirstConsumer` for volume topology aware dynamic provisioning:

```yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
  name: managed-disk-dynamic
parameters:
  cachingmode: None
  kind: Managed
  storageaccounttype: Standard_LRS
provisioner: kubernetes.io/azure-disk
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
allowedTopologies:
- matchLabelExpressions:
  - key: failure-domain.beta.kubernetes.io/zone
    values:
    - southeastasia-2
    - southeastasia-1
```

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
DynamicProvisioningScheduling and VolumeScheduling is not supported for Azure managed disks. Feature gates DynamicProvisioningScheduling and VolumeScheduling should be enabled before using this feature.
```

/kind feature
/sig azure
/cc @brendandburns @khenidak @andyzhangx
/cc @ddebroy @msau42 @justaugustus
This commit is contained in:
Kubernetes Submit Queue 2018-08-08 16:32:10 -07:00 committed by GitHub
commit ae351f1184
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 57 additions and 65 deletions

View File

@ -29,7 +29,6 @@ import (
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/sets"
kwait "k8s.io/apimachinery/pkg/util/wait" kwait "k8s.io/apimachinery/pkg/util/wait"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
"k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume"
@ -51,16 +50,8 @@ type ManagedDiskOptions struct {
PVCName string PVCName string
// The name of resource group. // The name of resource group.
ResourceGroup string ResourceGroup string
// Wether the disk is zoned.
Zoned bool
// Wether AvailabilityZone is set.
ZonePresent bool
// Wether AvailabilityZones is set.
ZonesPresent bool
// The AvailabilityZone to create the disk. // The AvailabilityZone to create the disk.
AvailabilityZone string AvailabilityZone string
// List of AvailabilityZone to create the disk.
AvailabilityZones string
// The tags of the disk. // The tags of the disk.
Tags map[string]string Tags map[string]string
// The SKU of storage account. // The SKU of storage account.
@ -73,44 +64,12 @@ func newManagedDiskController(common *controllerCommon) (*ManagedDiskController,
//CreateManagedDisk : create managed disk //CreateManagedDisk : create managed disk
func (c *ManagedDiskController) CreateManagedDisk(options *ManagedDiskOptions) (string, error) { func (c *ManagedDiskController) CreateManagedDisk(options *ManagedDiskOptions) (string, error) {
var zones sets.String
var activeZones sets.String
var err error var err error
glog.V(4).Infof("azureDisk - creating new managed Name:%s StorageAccountType:%s Size:%v", options.DiskName, options.StorageAccountType, options.SizeGB) glog.V(4).Infof("azureDisk - creating new managed Name:%s StorageAccountType:%s Size:%v", options.DiskName, options.StorageAccountType, options.SizeGB)
// Get active zones which have nodes running on.
activeZones, err = c.common.cloud.GetActiveZones()
if err != nil {
return "", fmt.Errorf("error querying active zones: %v", err)
}
// Validate and choose availability zone for creating disk.
if options.Zoned && !options.ZonePresent && !options.ZonesPresent {
// Neither "zone" or "zones" specified. Pick a zone randomly selected
// from all active zones where Kubernetes cluster has a node.
zones = activeZones
} else if !options.ZonePresent && options.ZonesPresent {
// Choose zone from specified zones.
if zones, err = util.ZonesToSet(options.AvailabilityZones); err != nil {
return "", err
}
} else if options.ZonePresent && !options.ZonesPresent {
if err := util.ValidateZone(options.AvailabilityZone); err != nil {
return "", err
}
zones = make(sets.String)
zones.Insert(options.AvailabilityZone)
}
var createZones *[]string var createZones *[]string
if len(zones.List()) > 0 { if len(options.AvailabilityZone) > 0 {
createAZ := util.ChooseZoneForVolume(zones, options.PVCName) zoneList := []string{c.common.cloud.GetZoneID(options.AvailabilityZone)}
// Do not allow creation of disks in zones that are do not have nodes. Such disks
// are not currently usable.
if !activeZones.Has(createAZ) {
return "", fmt.Errorf("kubernetes does not have a node in zone %q", createAZ)
}
zoneList := []string{c.common.cloud.GetZoneID(createAZ)}
createZones = &zoneList createZones = &zoneList
} }

View File

@ -22,9 +22,11 @@ import (
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute"
"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage"
"github.com/golang/glog" "github.com/golang/glog"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/pkg/cloudprovider/providers/azure" "k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
"k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util"
@ -61,6 +63,9 @@ type DiskController interface {
// GetAzureDiskLabels gets availability zone labels for Azuredisk. // GetAzureDiskLabels gets availability zone labels for Azuredisk.
GetAzureDiskLabels(diskURI string) (map[string]string, error) GetAzureDiskLabels(diskURI string) (map[string]string, error)
// GetActiveZones returns all the zones in which k8s nodes are currently running.
GetActiveZones() (sets.String, error)
} }
type azureDataDiskPlugin struct { type azureDataDiskPlugin struct {

View File

@ -25,6 +25,7 @@ import (
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/cloudprovider/providers/azure" "k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
"k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/features"
@ -123,7 +124,8 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
zonesPresent bool zonesPresent bool
strZoned string strZoned string
availabilityZone string availabilityZone string
availabilityZones string availabilityZones sets.String
selectedAvailabilityZone string
) )
// maxLength = 79 - (4 for ".vhd") = 75 // maxLength = 79 - (4 for ".vhd") = 75
name := util.GenerateVolumeName(p.options.ClusterName, p.options.PVName, 75) name := util.GenerateVolumeName(p.options.ClusterName, p.options.PVName, 75)
@ -156,7 +158,10 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
availabilityZone = v availabilityZone = v
case "zones": case "zones":
zonesPresent = true zonesPresent = true
availabilityZones = v availabilityZones, err = util.ZonesToSet(v)
if err != nil {
return nil, fmt.Errorf("error parsing zones %s, must be strings separated by commas: %v", v, err)
}
case "zoned": case "zoned":
strZoned = v strZoned = v
default: default:
@ -175,6 +180,16 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
return nil, err return nil, err
} }
if kind != v1.AzureManagedDisk {
if resourceGroup != "" {
return nil, errors.New("StorageClass option 'resourceGroup' can be used only for managed disks")
}
if zoned {
return nil, errors.New("StorageClass option 'zoned' parameter is only supported for managed disks")
}
}
zoned, err = parseZoned(strZoned, kind) zoned, err = parseZoned(strZoned, kind)
if err != nil { if err != nil {
return nil, err return nil, err
@ -184,10 +199,6 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
return nil, fmt.Errorf("zone or zones StorageClass parameters must be used together with zoned parameter") return nil, fmt.Errorf("zone or zones StorageClass parameters must be used together with zoned parameter")
} }
if zonePresent && zonesPresent {
return nil, fmt.Errorf("both zone and zones StorageClass parameters must not be used at the same time")
}
if cachingMode, err = normalizeCachingMode(cachingMode); err != nil { if cachingMode, err = normalizeCachingMode(cachingMode); err != nil {
return nil, err return nil, err
} }
@ -197,8 +208,17 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
return nil, err return nil, err
} }
if resourceGroup != "" && kind != v1.AzureManagedDisk { // Select zone for managed disks based on zone, zones and allowedTopologies.
return nil, errors.New("StorageClass option 'resourceGroup' can be used only for managed disks") if zoned {
activeZones, err := diskController.GetActiveZones()
if err != nil {
return nil, fmt.Errorf("error querying active zones: %v", err)
}
selectedAvailabilityZone, err = util.SelectZoneForVolume(zonePresent, zonesPresent, availabilityZone, availabilityZones, activeZones, selectedNode, allowedTopologies, p.options.PVC.Name)
if err != nil {
return nil, err
}
} }
// create disk // create disk
@ -217,11 +237,7 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
PVCName: p.options.PVC.Name, PVCName: p.options.PVC.Name,
SizeGB: requestGiB, SizeGB: requestGiB,
Tags: tags, Tags: tags,
Zoned: zoned, AvailabilityZone: selectedAvailabilityZone,
ZonePresent: zonePresent,
ZonesPresent: zonesPresent,
AvailabilityZone: availabilityZone,
AvailabilityZones: availabilityZones,
} }
diskURI, err = diskController.CreateManagedDisk(volumeOptions) diskURI, err = diskController.CreateManagedDisk(volumeOptions)
if err != nil { if err != nil {
@ -232,10 +248,6 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
return nil, err return nil, err
} }
} else { } else {
if zoned {
return nil, errors.New("zoned parameter is only supported for managed disks")
}
if kind == v1.AzureDedicatedBlobDisk { if kind == v1.AzureDedicatedBlobDisk {
_, diskURI, _, err = diskController.CreateVolume(name, account, storageAccountType, location, requestGiB) _, diskURI, _, err = diskController.CreateVolume(name, account, storageAccountType, location, requestGiB)
if err != nil { if err != nil {
@ -286,5 +298,21 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
}, },
} }
if zoned && utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
requirements := make([]v1.NodeSelectorRequirement, 0)
for k, v := range labels {
requirements = append(requirements, v1.NodeSelectorRequirement{Key: k, Operator: v1.NodeSelectorOpIn, Values: []string{v}})
}
nodeSelectorTerm := v1.NodeSelectorTerm{
MatchExpressions: requirements,
}
pv.Spec.NodeAffinity = &v1.VolumeNodeAffinity{
Required: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{nodeSelectorTerm},
},
}
}
return pv, nil return pv, nil
} }