From 1d527194656bad6a0f191f9fc6160bf7e931cf09 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Mon, 17 Oct 2016 14:50:00 +0000 Subject: [PATCH] azure disk volume: support storage class and dynamic provisioning Signed-off-by: Huamin Chen --- cmd/kube-controller-manager/app/BUILD | 1 + cmd/kube-controller-manager/app/plugins.go | 5 + .../persistent-volume-provisioning/README.md | 17 ++ pkg/cloudprovider/providers/azure/BUILD | 6 + pkg/kubectl/describe.go | 2 + pkg/volume/azure_dd/BUILD | 2 + pkg/volume/azure_dd/azure_dd.go | 21 ++- pkg/volume/azure_dd/azure_dd_test.go | 8 + pkg/volume/azure_dd/azure_provision.go | 162 ++++++++++++++++++ 9 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 pkg/volume/azure_dd/azure_provision.go diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index acdc721b5c8..f3c89ca9724 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -35,6 +35,7 @@ go_library( "//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider/providers:go_default_library", "//pkg/cloudprovider/providers/aws:go_default_library", + "//pkg/cloudprovider/providers/azure:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", "//pkg/cloudprovider/providers/openstack:go_default_library", "//pkg/cloudprovider/providers/vsphere:go_default_library", diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index accbbd3b203..0ba397e8e1e 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -31,6 +31,7 @@ import ( "github.com/golang/glog" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" + "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" @@ -121,6 +122,8 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...) case vsphere.ProviderName == cloud.ProviderName(): allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) + case azure.CloudProviderName == cloud.ProviderName(): + allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) } } @@ -149,6 +152,8 @@ func NewAlphaVolumeProvisioner(cloud cloudprovider.Interface, config componentco return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins()) case cloud != nil && vsphere.ProviderName == cloud.ProviderName(): return getProvisionablePluginFromVolumePlugins(vsphere_volume.ProbeVolumePlugins()) + case cloud != nil && azure.CloudProviderName == cloud.ProviderName(): + return getProvisionablePluginFromVolumePlugins(azure_dd.ProbeVolumePlugins()) } return nil, nil } diff --git a/examples/experimental/persistent-volume-provisioning/README.md b/examples/experimental/persistent-volume-provisioning/README.md index 2fce304d3d4..f387705c030 100644 --- a/examples/experimental/persistent-volume-provisioning/README.md +++ b/examples/experimental/persistent-volume-provisioning/README.md @@ -224,6 +224,23 @@ Create a Pod to use the PVC: $ kubectl create -f examples/experimental/persistent-volume-provisioning/quobyte/example-pod.yaml ``` +#### Azure Disk + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: slow +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Standard_LRS + location: eastus + storageAccount: azure_storage_account_name +``` + +* `skuName`: Azure storage account Sku tier. Default is empty. +* `location`: Azure storage account location. Default is empty. +* `storageAccount`: Azure storage account name. If storage account is not provided, all storage accounts associated with the resource group are searched to find one that matches `skuName` and `location`. If storage account is provided, `skuName` and `location` are ignored. ### User provisioning requests diff --git a/pkg/cloudprovider/providers/azure/BUILD b/pkg/cloudprovider/providers/azure/BUILD index 269cef8d017..143ba8d7c51 100644 --- a/pkg/cloudprovider/providers/azure/BUILD +++ b/pkg/cloudprovider/providers/azure/BUILD @@ -14,13 +14,16 @@ go_library( name = "go_default_library", srcs = [ "azure.go", + "azure_blob.go", "azure_instances.go", "azure_loadbalancer.go", "azure_routes.go", "azure_storage.go", + "azure_storageaccount.go", "azure_util.go", "azure_wrap.go", "azure_zones.go", + "vhd.go", ], tags = ["automanaged"], deps = [ @@ -30,11 +33,14 @@ go_library( "//pkg/util/errors:go_default_library", "//vendor:github.com/Azure/azure-sdk-for-go/arm/compute", "//vendor:github.com/Azure/azure-sdk-for-go/arm/network", + "//vendor:github.com/Azure/azure-sdk-for-go/arm/storage", + "//vendor:github.com/Azure/azure-sdk-for-go/storage", "//vendor:github.com/Azure/go-autorest/autorest", "//vendor:github.com/Azure/go-autorest/autorest/azure", "//vendor:github.com/Azure/go-autorest/autorest/to", "//vendor:github.com/ghodss/yaml", "//vendor:github.com/golang/glog", + "//vendor:github.com/rubiojr/go-vhd/vhd", ], ) diff --git a/pkg/kubectl/describe.go b/pkg/kubectl/describe.go index e4342d00218..e99dbac84b9 100644 --- a/pkg/kubectl/describe.go +++ b/pkg/kubectl/describe.go @@ -770,6 +770,8 @@ func (d *PersistentVolumeDescriber) Describe(namespace, name string, describerSe printVsphereVolumeSource(pv.Spec.VsphereVolume, out) case pv.Spec.Cinder != nil: printCinderVolumeSource(pv.Spec.Cinder, out) + case pv.Spec.AzureDisk != nil: + printAzureDiskVolumeSource(pv.Spec.AzureDisk, out) } if events != nil { diff --git a/pkg/volume/azure_dd/BUILD b/pkg/volume/azure_dd/BUILD index 277f6ce5ce8..60a0e513ac0 100644 --- a/pkg/volume/azure_dd/BUILD +++ b/pkg/volume/azure_dd/BUILD @@ -15,11 +15,13 @@ go_library( srcs = [ "attacher.go", "azure_dd.go", + "azure_provision.go", "vhd_util.go", ], tags = ["automanaged"], deps = [ "//pkg/api:go_default_library", + "//pkg/api/resource:go_default_library", "//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider/providers/azure:go_default_library", "//pkg/types:go_default_library", diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index 2ae776fbd29..d2a7732ecae 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -59,6 +59,10 @@ type azureCloudProvider interface { GetNextDiskLun(nodeName types.NodeName) (int32, error) // InstanceID returns the cloud provider ID of the specified instance. InstanceID(nodeName types.NodeName) (string, error) + // Create a VHD blob + CreateVolume(name, storageAccount, storageType, location string, requestGB int) (string, string, int, error) + // Delete a VHD blob + DeleteVolume(name, uri string) error } var _ volume.VolumePlugin = &azureDataDiskPlugin{} @@ -113,11 +117,20 @@ func (plugin *azureDataDiskPlugin) newMounterInternal(spec *volume.Spec, podUID if err != nil { return nil, err } - - fsType := *azure.FSType + fsType := "ext4" + if azure.FSType != nil { + fsType = *azure.FSType + } + cachingMode := api.AzureDataDiskCachingNone + if azure.CachingMode != nil { + cachingMode = *azure.CachingMode + } + readOnly := false + if azure.ReadOnly != nil { + readOnly = *azure.ReadOnly + } diskName := azure.DiskName diskUri := azure.DataDiskURI - cachingMode := *azure.CachingMode return &azureDiskMounter{ azureDisk: &azureDisk{ podUID: podUID, @@ -129,7 +142,7 @@ func (plugin *azureDataDiskPlugin) newMounterInternal(spec *volume.Spec, podUID plugin: plugin, }, fsType: fsType, - readOnly: *azure.ReadOnly, + readOnly: readOnly, diskMounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(), Runner: exec.New()}}, nil } diff --git a/pkg/volume/azure_dd/azure_dd_test.go b/pkg/volume/azure_dd/azure_dd_test.go index de8b5ba8e04..7450692b42e 100644 --- a/pkg/volume/azure_dd/azure_dd_test.go +++ b/pkg/volume/azure_dd/azure_dd_test.go @@ -91,6 +91,14 @@ func (fake *fakeAzureProvider) InstanceID(name string) (string, error) { return "localhost", nil } +func (fake *fakeAzureProvider) CreateVolume(name, storageAccount, storageType, location string, requestGB int) (string, string, int, error) { + return "", "", 0, fmt.Errorf("not implemented") +} + +func (fake *fakeAzureProvider) DeleteVolume(name, uri string) error { + return fmt.Errorf("not implemented") +} + func TestPlugin(t *testing.T) { tmpDir, err := utiltesting.MkTmpdir("azure_ddTest") if err != nil { diff --git a/pkg/volume/azure_dd/azure_provision.go b/pkg/volume/azure_dd/azure_provision.go new file mode 100644 index 00000000000..333e3950ef0 --- /dev/null +++ b/pkg/volume/azure_dd/azure_provision.go @@ -0,0 +1,162 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure_dd + +import ( + "fmt" + "strings" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + utilstrings "k8s.io/kubernetes/pkg/util/strings" + "k8s.io/kubernetes/pkg/volume" +) + +var _ volume.DeletableVolumePlugin = &azureDataDiskPlugin{} +var _ volume.ProvisionableVolumePlugin = &azureDataDiskPlugin{} + +type azureDiskDeleter struct { + *azureDisk + azureProvider azureCloudProvider +} + +func (plugin *azureDataDiskPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { + azure, err := getAzureCloudProvider(plugin.host.GetCloudProvider()) + if err != nil { + glog.V(4).Infof("failed to get azure provider") + return nil, err + } + + return plugin.newDeleterInternal(spec, azure) +} + +func (plugin *azureDataDiskPlugin) newDeleterInternal(spec *volume.Spec, azure azureCloudProvider) (volume.Deleter, error) { + if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.AzureDisk == nil { + return nil, fmt.Errorf("invalid PV spec") + } + diskName := spec.PersistentVolume.Spec.AzureDisk.DiskName + diskUri := spec.PersistentVolume.Spec.AzureDisk.DataDiskURI + return &azureDiskDeleter{ + azureDisk: &azureDisk{ + volName: spec.Name(), + diskName: diskName, + diskUri: diskUri, + plugin: plugin, + }, + azureProvider: azure, + }, nil +} + +func (plugin *azureDataDiskPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { + azure, err := getAzureCloudProvider(plugin.host.GetCloudProvider()) + if err != nil { + glog.V(4).Infof("failed to get azure provider") + return nil, err + } + if len(options.PVC.Spec.AccessModes) == 0 { + options.PVC.Spec.AccessModes = plugin.GetAccessModes() + } + return plugin.newProvisionerInternal(options, azure) +} + +func (plugin *azureDataDiskPlugin) newProvisionerInternal(options volume.VolumeOptions, azure azureCloudProvider) (volume.Provisioner, error) { + return &azureDiskProvisioner{ + azureDisk: &azureDisk{ + plugin: plugin, + }, + azureProvider: azure, + options: options, + }, nil +} + +var _ volume.Deleter = &azureDiskDeleter{} + +func (d *azureDiskDeleter) GetPath() string { + name := azureDataDiskPluginName + return d.plugin.host.GetPodVolumeDir(d.podUID, utilstrings.EscapeQualifiedNameForDisk(name), d.volName) +} + +func (d *azureDiskDeleter) Delete() error { + glog.V(4).Infof("deleting volume %s", d.diskUri) + return d.azureProvider.DeleteVolume(d.diskName, d.diskUri) +} + +type azureDiskProvisioner struct { + *azureDisk + azureProvider azureCloudProvider + options volume.VolumeOptions +} + +var _ volume.Provisioner = &azureDiskProvisioner{} + +func (a *azureDiskProvisioner) Provision() (*api.PersistentVolume, error) { + var sku, location, account string + + name := volume.GenerateVolumeName(a.options.ClusterName, a.options.PVName, 255) + capacity := a.options.PVC.Spec.Resources.Requests[api.ResourceName(api.ResourceStorage)] + requestBytes := capacity.Value() + requestGB := int(volume.RoundUpSize(requestBytes, 1024*1024*1024)) + + // Apply ProvisionerParameters (case-insensitive). We leave validation of + // the values to the cloud provider. + for k, v := range a.options.Parameters { + switch strings.ToLower(k) { + case "skuname": + sku = v + case "location": + location = v + case "storageaccount": + account = v + default: + return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, a.plugin.GetPluginName()) + } + } + // TODO: implement c.options.ProvisionerSelector parsing + if a.options.PVC.Spec.Selector != nil { + return nil, fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on Azure disk") + } + + diskName, diskUri, sizeGB, err := a.azureProvider.CreateVolume(name, account, sku, location, requestGB) + if err != nil { + return nil, err + } + + pv := &api.PersistentVolume{ + ObjectMeta: api.ObjectMeta{ + Name: a.options.PVName, + Labels: map[string]string{}, + Annotations: map[string]string{ + "kubernetes.io/createdby": "azure-disk-dynamic-provisioner", + }, + }, + Spec: api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: a.options.PersistentVolumeReclaimPolicy, + AccessModes: a.options.PVC.Spec.AccessModes, + Capacity: api.ResourceList{ + api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)), + }, + PersistentVolumeSource: api.PersistentVolumeSource{ + AzureDisk: &api.AzureDiskVolumeSource{ + DiskName: diskName, + DataDiskURI: diskUri, + }, + }, + }, + } + return pv, nil +}