diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index fbd0cdb6795..dbe78f0d17f 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -77,6 +77,7 @@ go_library( "//pkg/volume:go_default_library", "//pkg/volume/aws_ebs:go_default_library", "//pkg/volume/azure_dd:go_default_library", + "//pkg/volume/azure_file:go_default_library", "//pkg/volume/cinder:go_default_library", "//pkg/volume/flexvolume:go_default_library", "//pkg/volume/flocker:go_default_library", diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index e08c3e3b43f..a86e3336e0d 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -42,6 +42,7 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/aws_ebs" "k8s.io/kubernetes/pkg/volume/azure_dd" + "k8s.io/kubernetes/pkg/volume/azure_file" "k8s.io/kubernetes/pkg/volume/cinder" "k8s.io/kubernetes/pkg/volume/flexvolume" "k8s.io/kubernetes/pkg/volume/flocker" @@ -113,6 +114,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen // add rbd provisioner allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...) allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...) diff --git a/examples/persistent-volume-provisioning/README.md b/examples/persistent-volume-provisioning/README.md index 812e9d495f7..68863e6e5e6 100644 --- a/examples/persistent-volume-provisioning/README.md +++ b/examples/persistent-volume-provisioning/README.md @@ -242,7 +242,7 @@ Create a Pod to use the PVC: $ kubectl create -f examples/persistent-volume-provisioning/quobyte/example-pod.yaml ``` -#### Azure Disk +#### Azure Disk ```yaml kind: StorageClass @@ -260,6 +260,22 @@ parameters: * `location`: Azure storage account location. Default is empty. * `storageAccount`: Azure storage account name. If storage account is not provided, all storage accounts associated with the resource group are searched to find one that matches `skuName` and `location`. If storage account is provided, it must reside in the same resource group as the cluster, and `skuName` and `location` are ignored. +#### Azure File + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: slow +provisioner: kubernetes.io/azure-file +parameters: + skuName: Standard_LRS + location: eastus + storageAccount: azure_storage_account_name +``` + +The parameters are the same as those used by [Azure Disk](#azure-disk) + ### User provisioning requests Users request dynamically provisioned storage by including a storage class in their `PersistentVolumeClaim`. diff --git a/pkg/cloudprovider/providers/azure/BUILD b/pkg/cloudprovider/providers/azure/BUILD index 53b3631ec7e..12cd189c9b4 100644 --- a/pkg/cloudprovider/providers/azure/BUILD +++ b/pkg/cloudprovider/providers/azure/BUILD @@ -13,6 +13,7 @@ go_library( srcs = [ "azure.go", "azure_blob.go", + "azure_file.go", "azure_instances.go", "azure_loadbalancer.go", "azure_routes.go", diff --git a/pkg/cloudprovider/providers/azure/azure_file.go b/pkg/cloudprovider/providers/azure/azure_file.go new file mode 100644 index 00000000000..ccdca622a46 --- /dev/null +++ b/pkg/cloudprovider/providers/azure/azure_file.go @@ -0,0 +1,63 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "fmt" + "strconv" + + azs "github.com/Azure/azure-sdk-for-go/storage" +) + +// create file share +func (az *Cloud) createFileShare(accountName, accountKey, name string, sizeGB int) error { + fileClient, err := az.getFileSvcClient(accountName, accountKey) + if err != nil { + return err + } + // create a file share and set quota + // Note. Per https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Share, + // setting x-ms-share-quota can set quota on the new share, but in reality, setting quota in CreateShare + // receives error "The metadata specified is invalid. It has characters that are not permitted." + // As a result,breaking into two API calls: create share and set quota + if err = fileClient.CreateShare(name, nil); err != nil { + return fmt.Errorf("failed to create file share, err: %v", err) + } + if err = fileClient.SetShareProperties(name, azs.ShareHeaders{Quota: strconv.Itoa(sizeGB)}); err != nil { + az.deleteFileShare(accountName, accountKey, name) + return fmt.Errorf("failed to set quota on file share %s, err: %v", name, err) + } + return nil +} + +// delete a file share +func (az *Cloud) deleteFileShare(accountName, accountKey, name string) error { + fileClient, err := az.getFileSvcClient(accountName, accountKey) + if err == nil { + return fileClient.DeleteShare(name) + } + return err +} + +func (az *Cloud) getFileSvcClient(accountName, accountKey string) (*azs.FileServiceClient, error) { + client, err := azs.NewClient(accountName, accountKey, az.Environment.StorageEndpointSuffix, azs.DefaultAPIVersion, useHTTPS) + if err != nil { + return nil, fmt.Errorf("error creating azure client: %v", err) + } + f := client.GetFileService() + return &f, nil +} diff --git a/pkg/cloudprovider/providers/azure/azure_storage.go b/pkg/cloudprovider/providers/azure/azure_storage.go index c7c9ba4c9f0..abaae7c4019 100644 --- a/pkg/cloudprovider/providers/azure/azure_storage.go +++ b/pkg/cloudprovider/providers/azure/azure_storage.go @@ -251,3 +251,50 @@ func (az *Cloud) DeleteVolume(name, uri string) error { return nil } + +// CreateFileShare creates a file share, using a matching storage account +func (az *Cloud) CreateFileShare(name, storageAccount, storageType, location string, requestGB int) (string, string, error) { + var err error + accounts := []accountWithLocation{} + if len(storageAccount) > 0 { + accounts = append(accounts, accountWithLocation{Name: storageAccount}) + } else { + // find a storage account + accounts, err = az.getStorageAccounts() + if err != nil { + // TODO: create a storage account and container + return "", "", err + } + } + for _, account := range accounts { + glog.V(4).Infof("account %s type %s location %s", account.Name, account.StorageType, account.Location) + if ((storageType == "" || account.StorageType == storageType) && (location == "" || account.Location == location)) || len(storageAccount) > 0 { + // find the access key with this account + key, err := az.getStorageAccesskey(account.Name) + if err != nil { + glog.V(2).Infof("no key found for storage account %s", account.Name) + continue + } + + err = az.createFileShare(account.Name, key, name, requestGB) + if err != nil { + glog.V(2).Infof("failed to create share in account %s: %v", account.Name, err) + continue + } + glog.V(4).Infof("created share %s in account %s", name, account.Name) + return account.Name, key, err + } + } + return "", "", fmt.Errorf("failed to find a matching storage account") +} + +// DeleteFileShare deletes a file share using storage account name and key +func (az *Cloud) DeleteFileShare(accountName, key, name string) error { + err := az.deleteFileShare(accountName, key, name) + if err != nil { + return err + } + glog.V(4).Infof("share %s deleted", name) + return nil + +} diff --git a/pkg/volume/azure_file/BUILD b/pkg/volume/azure_file/BUILD index f851038eaaf..56666647708 100644 --- a/pkg/volume/azure_file/BUILD +++ b/pkg/volume/azure_file/BUILD @@ -12,17 +12,22 @@ go_library( name = "go_default_library", srcs = [ "azure_file.go", + "azure_provision.go", "azure_util.go", "doc.go", ], tags = ["automanaged"], deps = [ "//pkg/api/v1:go_default_library", + "//pkg/cloudprovider:go_default_library", + "//pkg/cloudprovider/providers/azure:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/util/strings:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", "//vendor:github.com/golang/glog", + "//vendor:k8s.io/apimachinery/pkg/api/errors", + "//vendor:k8s.io/apimachinery/pkg/api/resource", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", "//vendor:k8s.io/apimachinery/pkg/types", ], diff --git a/pkg/volume/azure_file/azure_file.go b/pkg/volume/azure_file/azure_file.go index 14ed10f2426..48063ddfc98 100644 --- a/pkg/volume/azure_file/azure_file.go +++ b/pkg/volume/azure_file/azure_file.go @@ -142,6 +142,7 @@ func (plugin *azureFilePlugin) ConstructVolumeSpec(volName, mountPath string) (* // azureFile volumes represent mount of an AzureFile share. type azureFile struct { volName string + podUID types.UID pod *v1.Pod mounter mount.Interface plugin *azureFilePlugin @@ -192,7 +193,7 @@ func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error { return nil } var accountKey, accountName string - if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.pod.Namespace, b.secretName, b.shareName); err != nil { + if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.pod.Namespace, b.secretName); err != nil { return err } os.MkdirAll(dir, 0750) diff --git a/pkg/volume/azure_file/azure_file_test.go b/pkg/volume/azure_file/azure_file_test.go index 9bf8bfd987e..8d3db99ff93 100644 --- a/pkg/volume/azure_file/azure_file_test.go +++ b/pkg/volume/azure_file/azure_file_test.go @@ -201,9 +201,12 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { type fakeAzureSvc struct{} -func (s *fakeAzureSvc) GetAzureCredentials(host volume.VolumeHost, nameSpace, secretName, shareName string) (string, string, error) { +func (s *fakeAzureSvc) GetAzureCredentials(host volume.VolumeHost, nameSpace, secretName string) (string, string, error) { return "name", "key", nil } +func (s *fakeAzureSvc) SetAzureCredentials(host volume.VolumeHost, nameSpace, accountName, accountKey string) (string, error) { + return "secret", nil +} func TestMounterAndUnmounterTypeAssert(t *testing.T) { tmpDir, err := ioutil.TempDir(os.TempDir(), "azurefileTest") diff --git a/pkg/volume/azure_file/azure_provision.go b/pkg/volume/azure_file/azure_provision.go new file mode 100644 index 00000000000..46a483e6b97 --- /dev/null +++ b/pkg/volume/azure_file/azure_provision.go @@ -0,0 +1,203 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure_file + +import ( + "fmt" + "strings" + + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" + utilstrings "k8s.io/kubernetes/pkg/util/strings" + "k8s.io/kubernetes/pkg/volume" +) + +var _ volume.DeletableVolumePlugin = &azureFilePlugin{} +var _ volume.ProvisionableVolumePlugin = &azureFilePlugin{} + +// Abstract interface to file share operations. +// azure cloud provider should implement it +type azureCloudProvider interface { + // create a file share + CreateFileShare(name, storageAccount, storageType, location string, requestGB int) (string, string, error) + // delete a file share + DeleteFileShare(accountName, key, name string) error +} + +type azureFileDeleter struct { + *azureFile + accountName, accountKey, shareName string + azureProvider azureCloudProvider +} + +func (plugin *azureFilePlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { + azure, err := getAzureCloudProvider(plugin.host.GetCloudProvider()) + if err != nil { + glog.V(4).Infof("failed to get azure provider") + return nil, err + } + + return plugin.newDeleterInternal(spec, &azureSvc{}, azure) +} + +func (plugin *azureFilePlugin) newDeleterInternal(spec *volume.Spec, util azureUtil, azure azureCloudProvider) (volume.Deleter, error) { + if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.AzureFile == nil { + return nil, fmt.Errorf("invalid PV spec") + } + pvSpec := spec.PersistentVolume + if pvSpec.Spec.ClaimRef.Namespace == "" { + glog.Errorf("namespace cannot be nil") + return nil, fmt.Errorf("invalid PV spec: nil namespace") + } + nameSpace := pvSpec.Spec.ClaimRef.Namespace + secretName := pvSpec.Spec.AzureFile.SecretName + shareName := pvSpec.Spec.AzureFile.ShareName + if accountName, accountKey, err := util.GetAzureCredentials(plugin.host, nameSpace, secretName); err != nil { + return nil, err + } else { + return &azureFileDeleter{ + azureFile: &azureFile{ + volName: spec.Name(), + plugin: plugin, + }, + shareName: shareName, + accountName: accountName, + accountKey: accountKey, + azureProvider: azure, + }, nil + } +} + +func (plugin *azureFilePlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { + azure, err := getAzureCloudProvider(plugin.host.GetCloudProvider()) + if err != nil { + glog.V(4).Infof("failed to get azure provider") + return nil, err + } + if len(options.PVC.Spec.AccessModes) == 0 { + options.PVC.Spec.AccessModes = plugin.GetAccessModes() + } + return plugin.newProvisionerInternal(options, azure) +} + +func (plugin *azureFilePlugin) newProvisionerInternal(options volume.VolumeOptions, azure azureCloudProvider) (volume.Provisioner, error) { + return &azureFileProvisioner{ + azureFile: &azureFile{ + plugin: plugin, + }, + azureProvider: azure, + util: &azureSvc{}, + options: options, + }, nil +} + +var _ volume.Deleter = &azureFileDeleter{} + +func (f *azureFileDeleter) GetPath() string { + name := azureFilePluginName + return f.plugin.host.GetPodVolumeDir(f.podUID, utilstrings.EscapeQualifiedNameForDisk(name), f.volName) +} + +func (f *azureFileDeleter) Delete() error { + glog.V(4).Infof("deleting volume %s", f.shareName) + return f.azureProvider.DeleteFileShare(f.accountName, f.accountKey, f.shareName) +} + +type azureFileProvisioner struct { + *azureFile + azureProvider azureCloudProvider + util azureUtil + options volume.VolumeOptions +} + +var _ volume.Provisioner = &azureFileProvisioner{} + +func (a *azureFileProvisioner) Provision() (*v1.PersistentVolume, error) { + var sku, location, account string + + name := volume.GenerateVolumeName(a.options.ClusterName, a.options.PVName, 75) + capacity := a.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] + requestBytes := capacity.Value() + requestGB := int(volume.RoundUpSize(requestBytes, 1024*1024*1024)) + + // Apply ProvisionerParameters (case-insensitive). We leave validation of + // the values to the cloud provider. + for k, v := range a.options.Parameters { + switch strings.ToLower(k) { + case "skuname": + sku = v + case "location": + location = v + case "storageaccount": + account = v + default: + return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, a.plugin.GetPluginName()) + } + } + // TODO: implement c.options.ProvisionerSelector parsing + if a.options.PVC.Spec.Selector != nil { + return nil, fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on Azure file") + } + + account, key, err := a.azureProvider.CreateFileShare(name, account, sku, location, requestGB) + if err != nil { + return nil, err + } + // create a secret for storage account and key + secretName, err := a.util.SetAzureCredentials(a.plugin.host, a.options.PVC.Namespace, account, key) + if err != nil { + return nil, err + } + // create PV + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.options.PVName, + Labels: map[string]string{}, + Annotations: map[string]string{ + "kubernetes.io/createdby": "azure-file-dynamic-provisioner", + }, + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: a.options.PersistentVolumeReclaimPolicy, + AccessModes: a.options.PVC.Spec.AccessModes, + Capacity: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", requestGB)), + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + AzureFile: &v1.AzureFileVolumeSource{ + SecretName: secretName, + ShareName: name, + }, + }, + }, + } + return pv, nil +} + +// Return cloud provider +func getAzureCloudProvider(cloudProvider cloudprovider.Interface) (azureCloudProvider, error) { + azureCloudProvider, ok := cloudProvider.(*azure.Cloud) + if !ok || azureCloudProvider == nil { + return nil, fmt.Errorf("Failed to get Azure Cloud Provider. GetCloudProvider returned %v instead", cloudProvider) + } + + return azureCloudProvider, nil +} diff --git a/pkg/volume/azure_file/azure_util.go b/pkg/volume/azure_file/azure_util.go index 9e2418d1fe3..dfe24273dc8 100644 --- a/pkg/volume/azure_file/azure_util.go +++ b/pkg/volume/azure_file/azure_util.go @@ -19,18 +19,21 @@ package azure_file import ( "fmt" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/volume" ) // Abstract interface to azure file operations. type azureUtil interface { - GetAzureCredentials(host volume.VolumeHost, nameSpace, secretName, shareName string) (string, string, error) + GetAzureCredentials(host volume.VolumeHost, nameSpace, secretName string) (string, string, error) + SetAzureCredentials(host volume.VolumeHost, nameSpace, accountName, accountKey string) (string, error) } type azureSvc struct{} -func (s *azureSvc) GetAzureCredentials(host volume.VolumeHost, nameSpace, secretName, shareName string) (string, string, error) { +func (s *azureSvc) GetAzureCredentials(host volume.VolumeHost, nameSpace, secretName string) (string, string, error) { var accountKey, accountName string kubeClient := host.GetKubeClient() if kubeClient == nil { @@ -54,3 +57,30 @@ func (s *azureSvc) GetAzureCredentials(host volume.VolumeHost, nameSpace, secret } return accountName, accountKey, nil } + +func (s *azureSvc) SetAzureCredentials(host volume.VolumeHost, nameSpace, accountName, accountKey string) (string, error) { + kubeClient := host.GetKubeClient() + if kubeClient == nil { + return "", fmt.Errorf("Cannot get kube client") + } + secretName := "azure-storage-account-" + accountName + "-secret" + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nameSpace, + Name: secretName, + }, + Data: map[string][]byte{ + "azurestorageaccountname": []byte(accountName), + "azurestorageaccountkey": []byte(accountKey), + }, + Type: "Opaque", + } + _, err := kubeClient.Core().Secrets(nameSpace).Create(secret) + if errors.IsAlreadyExists(err) { + err = nil + } + if err != nil { + return "", fmt.Errorf("Couldn't create secret %v", err) + } + return secretName, err +}