diff --git a/cmd/cloud-controller-manager/providers.go b/cmd/cloud-controller-manager/providers.go index b4ac9dbf337..6c6bb338305 100644 --- a/cmd/cloud-controller-manager/providers.go +++ b/cmd/cloud-controller-manager/providers.go @@ -25,6 +25,7 @@ package main import ( // NOTE: Importing all in-tree cloud-providers is not required when // implementing an out-of-tree cloud-provider. + _ "k8s.io/legacy-cloud-providers/azure" _ "k8s.io/legacy-cloud-providers/gce" _ "k8s.io/legacy-cloud-providers/vsphere" ) diff --git a/pkg/kubelet/cadvisor/cadvisor_cloudproviders.go b/pkg/kubelet/cadvisor/cadvisor_cloudproviders.go index d6dea8cf109..f9685f6fa81 100644 --- a/pkg/kubelet/cadvisor/cadvisor_cloudproviders.go +++ b/pkg/kubelet/cadvisor/cadvisor_cloudproviders.go @@ -22,5 +22,6 @@ package cadvisor import ( // Register cloud info providers. // TODO(#68522): Remove this in 1.20+ once the cAdvisor endpoints are removed. + _ "github.com/google/cadvisor/utils/cloudinfo/azure" _ "github.com/google/cadvisor/utils/cloudinfo/gce" ) diff --git a/pkg/volume/csimigration/plugin_manager.go b/pkg/volume/csimigration/plugin_manager.go index 41b6db90c98..2eacf54cb30 100644 --- a/pkg/volume/csimigration/plugin_manager.go +++ b/pkg/volume/csimigration/plugin_manager.go @@ -67,7 +67,7 @@ func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool { case csilibplugins.AzureFileInTreePluginName: return pm.featureGate.Enabled(features.InTreePluginAzureFileUnregister) case csilibplugins.AzureDiskInTreePluginName: - return true + return pm.featureGate.Enabled(features.InTreePluginAzureDiskUnregister) case csilibplugins.CinderInTreePluginName: return pm.featureGate.Enabled(features.InTreePluginOpenStackUnregister) case csilibplugins.VSphereInTreePluginName: diff --git a/plugin/pkg/admission/storage/persistentvolume/label/admission.go b/plugin/pkg/admission/storage/persistentvolume/label/admission.go index 23ee182080c..54dd507ace3 100644 --- a/plugin/pkg/admission/storage/persistentvolume/label/admission.go +++ b/plugin/pkg/admission/storage/persistentvolume/label/admission.go @@ -58,6 +58,7 @@ type persistentVolumeLabel struct { mutex sync.Mutex cloudConfig []byte gcePVLabeler cloudprovider.PVLabeler + azurePVLabeler cloudprovider.PVLabeler vspherePVLabeler cloudprovider.PVLabeler } @@ -70,7 +71,7 @@ var _ kubeapiserveradmission.WantsCloudConfig = &persistentVolumeLabel{} // As a side effect, the cloud provider may block invalid or non-existent volumes. func newPersistentVolumeLabel() *persistentVolumeLabel { // DEPRECATED: in a future release, we will use mutating admission webhooks to apply PV labels. - // Once the mutating admission webhook is used for GCE, + // Once the mutating admission webhook is used for Azure, and GCE, // this admission controller will be removed. klog.Warning("PersistentVolumeLabel admission controller is deprecated. " + "Please remove this controller from your configuration files and scripts.") @@ -204,6 +205,12 @@ func (l *persistentVolumeLabel) findVolumeLabels(volume *api.PersistentVolume) ( return nil, fmt.Errorf("error querying GCE PD volume %s: %v", volume.Spec.GCEPersistentDisk.PDName, err) } return labels, nil + case volume.Spec.AzureDisk != nil: + labels, err := l.findAzureDiskLabels(volume) + if err != nil { + return nil, fmt.Errorf("error querying AzureDisk volume %s: %v", volume.Spec.AzureDisk.DiskName, err) + } + return labels, nil case volume.Spec.VsphereVolume != nil: labels, err := l.findVsphereVolumeLabels(volume) if err != nil { @@ -264,6 +271,54 @@ func (l *persistentVolumeLabel) getGCEPVLabeler() (cloudprovider.PVLabeler, erro return l.gcePVLabeler, nil } +// getAzurePVLabeler returns the Azure implementation of PVLabeler +func (l *persistentVolumeLabel) getAzurePVLabeler() (cloudprovider.PVLabeler, error) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.azurePVLabeler == nil { + var cloudConfigReader io.Reader + if len(l.cloudConfig) > 0 { + cloudConfigReader = bytes.NewReader(l.cloudConfig) + } + + cloudProvider, err := cloudprovider.GetCloudProvider("azure", cloudConfigReader) + if err != nil || cloudProvider == nil { + return nil, err + } + + azurePVLabeler, ok := cloudProvider.(cloudprovider.PVLabeler) + if !ok { + return nil, errors.New("Azure cloud provider does not implement PV labeling") + } + l.azurePVLabeler = azurePVLabeler + } + + return l.azurePVLabeler, nil +} + +func (l *persistentVolumeLabel) findAzureDiskLabels(volume *api.PersistentVolume) (map[string]string, error) { + // Ignore any volumes that are being provisioned + if volume.Spec.AzureDisk.DiskName == cloudvolume.ProvisionedVolumeName { + return nil, nil + } + + pvlabler, err := l.getAzurePVLabeler() + if err != nil { + return nil, err + } + if pvlabler == nil { + return nil, fmt.Errorf("unable to build Azure cloud provider for AzureDisk") + } + + pv := &v1.PersistentVolume{} + err = k8s_api_v1.Convert_core_PersistentVolume_To_v1_PersistentVolume(volume, pv, nil) + if err != nil { + return nil, fmt.Errorf("failed to convert PersistentVolume to core/v1: %q", err) + } + return pvlabler.GetLabelsForVolume(context.TODO(), pv) +} + func (l *persistentVolumeLabel) findVsphereVolumeLabels(volume *api.PersistentVolume) (map[string]string, error) { pvlabler, err := l.getVspherePVLabeler() if err != nil { diff --git a/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go b/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go index 7db6ff0f187..b91172d0df4 100644 --- a/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go +++ b/plugin/pkg/admission/storage/persistentvolume/label/admission_test.go @@ -431,6 +431,72 @@ func Test_PVLAdmission(t *testing.T) { }, err: nil, }, + { + name: "Azure Disk PV labeled correctly", + handler: newPersistentVolumeLabel(), + pvlabeler: mockVolumeLabels(map[string]string{ + "a": "1", + "b": "2", + v1.LabelFailureDomainBetaZone: "1__2__3", + }), + preAdmissionPV: &api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azurepd", + Namespace: "myns", + }, + Spec: api.PersistentVolumeSpec{ + PersistentVolumeSource: api.PersistentVolumeSource{ + AzureDisk: &api.AzureDiskVolumeSource{ + DiskName: "123", + }, + }, + }, + }, + postAdmissionPV: &api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azurepd", + Namespace: "myns", + Labels: map[string]string{ + "a": "1", + "b": "2", + v1.LabelFailureDomainBetaZone: "1__2__3", + }, + }, + Spec: api.PersistentVolumeSpec{ + PersistentVolumeSource: api.PersistentVolumeSource{ + AzureDisk: &api.AzureDiskVolumeSource{ + DiskName: "123", + }, + }, + NodeAffinity: &api.VolumeNodeAffinity{ + Required: &api.NodeSelector{ + NodeSelectorTerms: []api.NodeSelectorTerm{ + { + MatchExpressions: []api.NodeSelectorRequirement{ + { + Key: "a", + Operator: api.NodeSelectorOpIn, + Values: []string{"1"}, + }, + { + Key: "b", + Operator: api.NodeSelectorOpIn, + Values: []string{"2"}, + }, + { + Key: v1.LabelFailureDomainBetaZone, + Operator: api.NodeSelectorOpIn, + Values: []string{"1", "2", "3"}, + }, + }, + }, + }, + }, + }, + }, + }, + err: nil, + }, { name: "vSphere PV non-conflicting affinity rules added", handler: newPersistentVolumeLabel(), @@ -640,9 +706,10 @@ func Test_PVLAdmission(t *testing.T) { // setPVLabler applies the given mock pvlabeler to implement PV labeling for all cloud providers. // Given we mock out the values of the labels anyways, assigning the same mock labeler for every // provider does not reduce test coverage but it does simplify/clean up the tests here because -// the provider is then decided based on the type of PV (EBS, GCEPD, etc) +// the provider is then decided based on the type of PV (EBS, GCEPD, Azure Disk, etc) func setPVLabeler(handler *persistentVolumeLabel, pvlabeler cloudprovider.PVLabeler) { handler.gcePVLabeler = pvlabeler + handler.azurePVLabeler = pvlabeler handler.vspherePVLabeler = pvlabeler } diff --git a/test/e2e/framework/providers/azure/azure.go b/test/e2e/framework/providers/azure/azure.go index 03f6261f01f..7c6bb217dfa 100644 --- a/test/e2e/framework/providers/azure/azure.go +++ b/test/e2e/framework/providers/azure/azure.go @@ -63,6 +63,28 @@ func (p *Provider) DeleteNode(node *v1.Node) error { return errors.New("not implemented yet") } +// CreatePD creates a persistent volume +func (p *Provider) CreatePD(zone string) (string, error) { + pdName := fmt.Sprintf("%s-%s", framework.TestContext.Prefix, string(uuid.NewUUID())) + + volumeOptions := &azure.ManagedDiskOptions{ + DiskName: pdName, + StorageAccountType: compute.StandardLRS, + ResourceGroup: "", + PVCName: pdName, + SizeGB: 1, + Tags: nil, + DiskIOPSReadWrite: "", + DiskMBpsReadWrite: "", + } + + // do not use blank zone definition + if len(zone) > 0 { + volumeOptions.AvailabilityZone = zone + } + return p.azureCloud.CreateManagedDisk(volumeOptions) +} + // CreateShare creates a share and return its account name and key. func (p *Provider) CreateShare() (string, string, string, error) { accountOptions := &azure.AccountOptions{ @@ -96,6 +118,15 @@ func (p *Provider) DeleteShare(accountName, shareName string) error { return err } +// DeletePD deletes a persistent volume +func (p *Provider) DeletePD(pdName string) error { + if err := p.azureCloud.DeleteManagedDisk(pdName); err != nil { + framework.Logf("failed to delete Azure volume %q: %v", pdName, err) + return err + } + return nil +} + // EnableAndDisableInternalLB returns functions for both enabling and disabling internal Load Balancer func (p *Provider) EnableAndDisableInternalLB() (enable, disable func(svc *v1.Service)) { enable = func(svc *v1.Service) { diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 11f83b04cb5..573bd4b7a95 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -40,6 +40,7 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" @@ -1313,6 +1314,150 @@ func (v *vSphereVolume) DeleteVolume(ctx context.Context) { v.nodeInfo.VSphere.DeleteVolume(v.volumePath, v.nodeInfo.DataCenterRef) } +// Azure Disk +type azureDiskDriver struct { + driverInfo storageframework.DriverInfo +} + +type azureDiskVolume struct { + volumeName string +} + +var _ storageframework.TestDriver = &azureDiskDriver{} +var _ storageframework.PreprovisionedVolumeTestDriver = &azureDiskDriver{} +var _ storageframework.InlineVolumeTestDriver = &azureDiskDriver{} +var _ storageframework.PreprovisionedPVTestDriver = &azureDiskDriver{} +var _ storageframework.DynamicPVTestDriver = &azureDiskDriver{} +var _ storageframework.CustomTimeoutsTestDriver = &azureDiskDriver{} + +// InitAzureDiskDriver returns azureDiskDriver that implements TestDriver interface +func InitAzureDiskDriver() storageframework.TestDriver { + return &azureDiskDriver{ + driverInfo: storageframework.DriverInfo{ + Name: "azure-disk", + InTreePluginName: "kubernetes.io/azure-disk", + MaxFileSize: storageframework.FileSizeMedium, + SupportedSizeRange: e2evolume.SizeRange{ + Min: "1Gi", + }, + SupportedFsType: sets.NewString( + "", // Default fsType + "ext4", + "xfs", + ), + TopologyKeys: []string{v1.LabelFailureDomainBetaZone}, + Capabilities: map[storageframework.Capability]bool{ + storageframework.CapPersistence: true, + storageframework.CapFsGroup: true, + storageframework.CapBlock: true, + storageframework.CapExec: true, + storageframework.CapMultiPODs: true, + // Azure supports volume limits, but the test creates large + // number of volumes and times out test suites. + storageframework.CapVolumeLimits: false, + storageframework.CapTopology: true, + storageframework.CapMultiplePVsSameID: true, + }, + }, + } +} + +func (a *azureDiskDriver) GetDriverInfo() *storageframework.DriverInfo { + return &a.driverInfo +} + +func (a *azureDiskDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) { + e2eskipper.SkipUnlessProviderIs("azure") +} + +func (a *azureDiskDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource { + av, ok := e2evolume.(*azureDiskVolume) + if !ok { + framework.Failf("Failed to cast test volume of type %T to the Azure test volume", e2evolume) + } + diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):] + + kind := v1.AzureManagedDisk + volSource := v1.VolumeSource{ + AzureDisk: &v1.AzureDiskVolumeSource{ + DiskName: diskName, + DataDiskURI: av.volumeName, + Kind: &kind, + ReadOnly: &readOnly, + }, + } + if fsType != "" { + volSource.AzureDisk.FSType = &fsType + } + return &volSource +} + +func (a *azureDiskDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) { + av, ok := e2evolume.(*azureDiskVolume) + if !ok { + framework.Failf("Failed to cast test volume of type %T to the Azure test volume", e2evolume) + } + + diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):] + + kind := v1.AzureManagedDisk + pvSource := v1.PersistentVolumeSource{ + AzureDisk: &v1.AzureDiskVolumeSource{ + DiskName: diskName, + DataDiskURI: av.volumeName, + Kind: &kind, + ReadOnly: &readOnly, + }, + } + if fsType != "" { + pvSource.AzureDisk.FSType = &fsType + } + return &pvSource, nil +} + +func (a *azureDiskDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass { + provisioner := "kubernetes.io/azure-disk" + parameters := map[string]string{} + if fsType != "" { + parameters["fsType"] = fsType + } + ns := config.Framework.Namespace.Name + delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer + + return storageframework.GetStorageClass(provisioner, parameters, &delayedBinding, ns) +} + +func (a *azureDiskDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig { + return &storageframework.PerTestConfig{ + Driver: a, + Prefix: "azure", + Framework: f, + } +} + +func (a *azureDiskDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume { + ginkgo.By("creating a test azure disk volume") + zone := getInlineVolumeZone(ctx, config.Framework) + if volType == storageframework.InlineVolume { + // PD will be created in framework.TestContext.CloudConfig.Zone zone, + // so pods should be also scheduled there. + config.ClientNodeSelection = e2epod.NodeSelection{ + Selector: map[string]string{ + v1.LabelFailureDomainBetaZone: zone, + }, + } + } + volumeName, err := e2epv.CreatePDWithRetryAndZone(ctx, zone) + framework.ExpectNoError(err) + return &azureDiskVolume{ + volumeName: volumeName, + } +} + +func (v *azureDiskVolume) DeleteVolume(ctx context.Context) { + _ = e2epv.DeletePDWithRetry(ctx, v.volumeName) +} + // AWS type awsDriver struct { driverInfo storageframework.DriverInfo @@ -1755,3 +1900,11 @@ func (v *azureFileVolume) DeleteVolume(ctx context.Context) { err := e2epv.DeleteShare(v.accountName, v.shareName) framework.ExpectNoError(err) } + +func (a *azureDiskDriver) GetTimeouts() *framework.TimeoutContext { + timeouts := framework.NewTimeoutContext() + timeouts.PodStart = time.Minute * 15 + timeouts.PodDelete = time.Minute * 15 + timeouts.PVDelete = time.Minute * 20 + return timeouts +} diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go index 5b81c682580..53764bcbc5c 100644 --- a/test/e2e/storage/in_tree_volumes.go +++ b/test/e2e/storage/in_tree_volumes.go @@ -38,6 +38,7 @@ var testDrivers = []func() storageframework.TestDriver{ drivers.InitEmptydirDriver, drivers.InitCinderDriver, drivers.InitVSphereDriver, + drivers.InitAzureDiskDriver, drivers.InitAzureFileDriver, drivers.InitAwsDriver, drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectory), diff --git a/vendor/github.com/google/cadvisor/utils/cloudinfo/azure/azure.go b/vendor/github.com/google/cadvisor/utils/cloudinfo/azure/azure.go new file mode 100644 index 00000000000..d04b0e2b404 --- /dev/null +++ b/vendor/github.com/google/cadvisor/utils/cloudinfo/azure/azure.go @@ -0,0 +1,58 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudinfo + +import ( + "io/ioutil" + "strings" + + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/utils/cloudinfo" +) + +const ( + sysVendorFileName = "/sys/class/dmi/id/sys_vendor" + biosUUIDFileName = "/sys/class/dmi/id/product_uuid" + microsoftCorporation = "Microsoft Corporation" +) + +func init() { + cloudinfo.RegisterCloudProvider(info.Azure, &provider{}) +} + +type provider struct{} + +var _ cloudinfo.CloudProvider = provider{} + +func (provider) IsActiveProvider() bool { + data, err := ioutil.ReadFile(sysVendorFileName) + if err != nil { + return false + } + return strings.Contains(string(data), microsoftCorporation) +} + +// TODO: Implement method. +func (provider) GetInstanceType() info.InstanceType { + return info.UnknownInstance +} + +func (provider) GetInstanceID() info.InstanceID { + data, err := ioutil.ReadFile(biosUUIDFileName) + if err != nil { + return info.UnNamedInstance + } + return info.InstanceID(strings.TrimSuffix(string(data), "\n")) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c1f35f49804..fc9f4b144e4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -384,6 +384,7 @@ github.com/google/cadvisor/third_party/containerd/api/types github.com/google/cadvisor/third_party/containerd/api/types/task github.com/google/cadvisor/utils github.com/google/cadvisor/utils/cloudinfo +github.com/google/cadvisor/utils/cloudinfo/azure github.com/google/cadvisor/utils/cloudinfo/gce github.com/google/cadvisor/utils/cpuload github.com/google/cadvisor/utils/cpuload/netlink