revert azuredisk test removal change

revert

revert vendor changes

revert

revert

fix

cmd/cloud-controller-manager/providers.go

@@ -25,6 +25,7 @@ package main
 import (
 	// NOTE: Importing all in-tree cloud-providers is not required when
 	// implementing an out-of-tree cloud-provider.
+	_ "k8s.io/legacy-cloud-providers/azure"
 	_ "k8s.io/legacy-cloud-providers/gce"
 	_ "k8s.io/legacy-cloud-providers/vsphere"
 )

pkg/kubelet/cadvisor/cadvisor_cloudproviders.go

@@ -22,5 +22,6 @@ package cadvisor
 import (
 	// Register cloud info providers.
 	// TODO(#68522): Remove this in 1.20+ once the cAdvisor endpoints are removed.
+	_ "github.com/google/cadvisor/utils/cloudinfo/azure"
 	_ "github.com/google/cadvisor/utils/cloudinfo/gce"
 )

pkg/volume/csimigration/plugin_manager.go

@@ -67,7 +67,7 @@ func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool {
 	case csilibplugins.AzureFileInTreePluginName:
 		return pm.featureGate.Enabled(features.InTreePluginAzureFileUnregister)
 	case csilibplugins.AzureDiskInTreePluginName:
-		return true
+		return pm.featureGate.Enabled(features.InTreePluginAzureDiskUnregister)
 	case csilibplugins.CinderInTreePluginName:
 		return pm.featureGate.Enabled(features.InTreePluginOpenStackUnregister)
 	case csilibplugins.VSphereInTreePluginName:

plugin/pkg/admission/storage/persistentvolume/label/admission.go

@@ -58,6 +58,7 @@ type persistentVolumeLabel struct {
 	mutex            sync.Mutex
 	cloudConfig      []byte
 	gcePVLabeler     cloudprovider.PVLabeler
+	azurePVLabeler   cloudprovider.PVLabeler
 	vspherePVLabeler cloudprovider.PVLabeler
 }
 
@@ -70,7 +71,7 @@ var _ kubeapiserveradmission.WantsCloudConfig = &persistentVolumeLabel{}
 // As a side effect, the cloud provider may block invalid or non-existent volumes.
 func newPersistentVolumeLabel() *persistentVolumeLabel {
 	// DEPRECATED: in a future release, we will use mutating admission webhooks to apply PV labels.
-	// Once the mutating admission webhook is used for GCE,
+	// Once the mutating admission webhook is used for Azure, and GCE,
 	// this admission controller will be removed.
 	klog.Warning("PersistentVolumeLabel admission controller is deprecated. " +
 		"Please remove this controller from your configuration files and scripts.")
@@ -204,6 +205,12 @@ func (l *persistentVolumeLabel) findVolumeLabels(volume *api.PersistentVolume) (
 			return nil, fmt.Errorf("error querying GCE PD volume %s: %v", volume.Spec.GCEPersistentDisk.PDName, err)
 		}
 		return labels, nil
+	case volume.Spec.AzureDisk != nil:
+		labels, err := l.findAzureDiskLabels(volume)
+		if err != nil {
+			return nil, fmt.Errorf("error querying AzureDisk volume %s: %v", volume.Spec.AzureDisk.DiskName, err)
+		}
+		return labels, nil
 	case volume.Spec.VsphereVolume != nil:
 		labels, err := l.findVsphereVolumeLabels(volume)
 		if err != nil {
@@ -264,6 +271,54 @@ func (l *persistentVolumeLabel) getGCEPVLabeler() (cloudprovider.PVLabeler, erro
 	return l.gcePVLabeler, nil
 }
 
+// getAzurePVLabeler returns the Azure implementation of PVLabeler
+func (l *persistentVolumeLabel) getAzurePVLabeler() (cloudprovider.PVLabeler, error) {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+
+	if l.azurePVLabeler == nil {
+		var cloudConfigReader io.Reader
+		if len(l.cloudConfig) > 0 {
+			cloudConfigReader = bytes.NewReader(l.cloudConfig)
+		}
+
+		cloudProvider, err := cloudprovider.GetCloudProvider("azure", cloudConfigReader)
+		if err != nil || cloudProvider == nil {
+			return nil, err
+		}
+
+		azurePVLabeler, ok := cloudProvider.(cloudprovider.PVLabeler)
+		if !ok {
+			return nil, errors.New("Azure cloud provider does not implement PV labeling")
+		}
+		l.azurePVLabeler = azurePVLabeler
+	}
+
+	return l.azurePVLabeler, nil
+}
+
+func (l *persistentVolumeLabel) findAzureDiskLabels(volume *api.PersistentVolume) (map[string]string, error) {
+	// Ignore any volumes that are being provisioned
+	if volume.Spec.AzureDisk.DiskName == cloudvolume.ProvisionedVolumeName {
+		return nil, nil
+	}
+
+	pvlabler, err := l.getAzurePVLabeler()
+	if err != nil {
+		return nil, err
+	}
+	if pvlabler == nil {
+		return nil, fmt.Errorf("unable to build Azure cloud provider for AzureDisk")
+	}
+
+	pv := &v1.PersistentVolume{}
+	err = k8s_api_v1.Convert_core_PersistentVolume_To_v1_PersistentVolume(volume, pv, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert PersistentVolume to core/v1: %q", err)
+	}
+	return pvlabler.GetLabelsForVolume(context.TODO(), pv)
+}
+
 func (l *persistentVolumeLabel) findVsphereVolumeLabels(volume *api.PersistentVolume) (map[string]string, error) {
 	pvlabler, err := l.getVspherePVLabeler()
 	if err != nil {

plugin/pkg/admission/storage/persistentvolume/label/admission_test.go

@@ -431,6 +431,72 @@ func Test_PVLAdmission(t *testing.T) {
 			},
 			err: nil,
 		},
+		{
+			name:    "Azure Disk PV labeled correctly",
+			handler: newPersistentVolumeLabel(),
+			pvlabeler: mockVolumeLabels(map[string]string{
+				"a":                           "1",
+				"b":                           "2",
+				v1.LabelFailureDomainBetaZone: "1__2__3",
+			}),
+			preAdmissionPV: &api.PersistentVolume{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "azurepd",
+					Namespace: "myns",
+				},
+				Spec: api.PersistentVolumeSpec{
+					PersistentVolumeSource: api.PersistentVolumeSource{
+						AzureDisk: &api.AzureDiskVolumeSource{
+							DiskName: "123",
+						},
+					},
+				},
+			},
+			postAdmissionPV: &api.PersistentVolume{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "azurepd",
+					Namespace: "myns",
+					Labels: map[string]string{
+						"a":                           "1",
+						"b":                           "2",
+						v1.LabelFailureDomainBetaZone: "1__2__3",
+					},
+				},
+				Spec: api.PersistentVolumeSpec{
+					PersistentVolumeSource: api.PersistentVolumeSource{
+						AzureDisk: &api.AzureDiskVolumeSource{
+							DiskName: "123",
+						},
+					},
+					NodeAffinity: &api.VolumeNodeAffinity{
+						Required: &api.NodeSelector{
+							NodeSelectorTerms: []api.NodeSelectorTerm{
+								{
+									MatchExpressions: []api.NodeSelectorRequirement{
+										{
+											Key:      "a",
+											Operator: api.NodeSelectorOpIn,
+											Values:   []string{"1"},
+										},
+										{
+											Key:      "b",
+											Operator: api.NodeSelectorOpIn,
+											Values:   []string{"2"},
+										},
+										{
+											Key:      v1.LabelFailureDomainBetaZone,
+											Operator: api.NodeSelectorOpIn,
+											Values:   []string{"1", "2", "3"},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			err: nil,
+		},
 		{
 			name:    "vSphere PV non-conflicting affinity rules added",
 			handler: newPersistentVolumeLabel(),
@@ -640,9 +706,10 @@ func Test_PVLAdmission(t *testing.T) {
 // setPVLabler applies the given mock pvlabeler to implement PV labeling for all cloud providers.
 // Given we mock out the values of the labels anyways, assigning the same mock labeler for every
 // provider does not reduce test coverage but it does simplify/clean up the tests here because
-// the provider is then decided based on the type of PV (EBS, GCEPD, etc)
+// the provider is then decided based on the type of PV (EBS, GCEPD, Azure Disk, etc)
 func setPVLabeler(handler *persistentVolumeLabel, pvlabeler cloudprovider.PVLabeler) {
 	handler.gcePVLabeler = pvlabeler
+	handler.azurePVLabeler = pvlabeler
 	handler.vspherePVLabeler = pvlabeler
 }
 

test/e2e/framework/providers/azure/azure.go

@@ -63,6 +63,28 @@ func (p *Provider) DeleteNode(node *v1.Node) error {
 	return errors.New("not implemented yet")
 }
 
+// CreatePD creates a persistent volume
+func (p *Provider) CreatePD(zone string) (string, error) {
+	pdName := fmt.Sprintf("%s-%s", framework.TestContext.Prefix, string(uuid.NewUUID()))
+
+	volumeOptions := &azure.ManagedDiskOptions{
+		DiskName:           pdName,
+		StorageAccountType: compute.StandardLRS,
+		ResourceGroup:      "",
+		PVCName:            pdName,
+		SizeGB:             1,
+		Tags:               nil,
+		DiskIOPSReadWrite:  "",
+		DiskMBpsReadWrite:  "",
+	}
+
+	// do not use blank zone definition
+	if len(zone) > 0 {
+		volumeOptions.AvailabilityZone = zone
+	}
+	return p.azureCloud.CreateManagedDisk(volumeOptions)
+}
+
 // CreateShare creates a share and return its account name and key.
 func (p *Provider) CreateShare() (string, string, string, error) {
 	accountOptions := &azure.AccountOptions{
@@ -96,6 +118,15 @@ func (p *Provider) DeleteShare(accountName, shareName string) error {
 	return err
 }
 
+// DeletePD deletes a persistent volume
+func (p *Provider) DeletePD(pdName string) error {
+	if err := p.azureCloud.DeleteManagedDisk(pdName); err != nil {
+		framework.Logf("failed to delete Azure volume %q: %v", pdName, err)
+		return err
+	}
+	return nil
+}
+
 // EnableAndDisableInternalLB returns functions for both enabling and disabling internal Load Balancer
 func (p *Provider) EnableAndDisableInternalLB() (enable, disable func(svc *v1.Service)) {
 	enable = func(svc *v1.Service) {

test/e2e/storage/drivers/in_tree.go

@@ -40,6 +40,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	v1 "k8s.io/api/core/v1"
@@ -1313,6 +1314,150 @@ func (v *vSphereVolume) DeleteVolume(ctx context.Context) {
 	v.nodeInfo.VSphere.DeleteVolume(v.volumePath, v.nodeInfo.DataCenterRef)
 }
 
+// Azure Disk
+type azureDiskDriver struct {
+	driverInfo storageframework.DriverInfo
+}
+
+type azureDiskVolume struct {
+	volumeName string
+}
+
+var _ storageframework.TestDriver = &azureDiskDriver{}
+var _ storageframework.PreprovisionedVolumeTestDriver = &azureDiskDriver{}
+var _ storageframework.InlineVolumeTestDriver = &azureDiskDriver{}
+var _ storageframework.PreprovisionedPVTestDriver = &azureDiskDriver{}
+var _ storageframework.DynamicPVTestDriver = &azureDiskDriver{}
+var _ storageframework.CustomTimeoutsTestDriver = &azureDiskDriver{}
+
+// InitAzureDiskDriver returns azureDiskDriver that implements TestDriver interface
+func InitAzureDiskDriver() storageframework.TestDriver {
+	return &azureDiskDriver{
+		driverInfo: storageframework.DriverInfo{
+			Name:             "azure-disk",
+			InTreePluginName: "kubernetes.io/azure-disk",
+			MaxFileSize:      storageframework.FileSizeMedium,
+			SupportedSizeRange: e2evolume.SizeRange{
+				Min: "1Gi",
+			},
+			SupportedFsType: sets.NewString(
+				"", // Default fsType
+				"ext4",
+				"xfs",
+			),
+			TopologyKeys: []string{v1.LabelFailureDomainBetaZone},
+			Capabilities: map[storageframework.Capability]bool{
+				storageframework.CapPersistence: true,
+				storageframework.CapFsGroup:     true,
+				storageframework.CapBlock:       true,
+				storageframework.CapExec:        true,
+				storageframework.CapMultiPODs:   true,
+				// Azure supports volume limits, but the test creates large
+				// number of volumes and times out test suites.
+				storageframework.CapVolumeLimits:      false,
+				storageframework.CapTopology:          true,
+				storageframework.CapMultiplePVsSameID: true,
+			},
+		},
+	}
+}
+
+func (a *azureDiskDriver) GetDriverInfo() *storageframework.DriverInfo {
+	return &a.driverInfo
+}
+
+func (a *azureDiskDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
+	e2eskipper.SkipUnlessProviderIs("azure")
+}
+
+func (a *azureDiskDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
+	av, ok := e2evolume.(*azureDiskVolume)
+	if !ok {
+		framework.Failf("Failed to cast test volume of type %T to the Azure test volume", e2evolume)
+	}
+	diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
+
+	kind := v1.AzureManagedDisk
+	volSource := v1.VolumeSource{
+		AzureDisk: &v1.AzureDiskVolumeSource{
+			DiskName:    diskName,
+			DataDiskURI: av.volumeName,
+			Kind:        &kind,
+			ReadOnly:    &readOnly,
+		},
+	}
+	if fsType != "" {
+		volSource.AzureDisk.FSType = &fsType
+	}
+	return &volSource
+}
+
+func (a *azureDiskDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+	av, ok := e2evolume.(*azureDiskVolume)
+	if !ok {
+		framework.Failf("Failed to cast test volume of type %T to the Azure test volume", e2evolume)
+	}
+
+	diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
+
+	kind := v1.AzureManagedDisk
+	pvSource := v1.PersistentVolumeSource{
+		AzureDisk: &v1.AzureDiskVolumeSource{
+			DiskName:    diskName,
+			DataDiskURI: av.volumeName,
+			Kind:        &kind,
+			ReadOnly:    &readOnly,
+		},
+	}
+	if fsType != "" {
+		pvSource.AzureDisk.FSType = &fsType
+	}
+	return &pvSource, nil
+}
+
+func (a *azureDiskDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
+	provisioner := "kubernetes.io/azure-disk"
+	parameters := map[string]string{}
+	if fsType != "" {
+		parameters["fsType"] = fsType
+	}
+	ns := config.Framework.Namespace.Name
+	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
+
+	return storageframework.GetStorageClass(provisioner, parameters, &delayedBinding, ns)
+}
+
+func (a *azureDiskDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
+	return &storageframework.PerTestConfig{
+		Driver:    a,
+		Prefix:    "azure",
+		Framework: f,
+	}
+}
+
+func (a *azureDiskDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
+	ginkgo.By("creating a test azure disk volume")
+	zone := getInlineVolumeZone(ctx, config.Framework)
+	if volType == storageframework.InlineVolume {
+		// PD will be created in framework.TestContext.CloudConfig.Zone zone,
+		// so pods should be also scheduled there.
+		config.ClientNodeSelection = e2epod.NodeSelection{
+			Selector: map[string]string{
+				v1.LabelFailureDomainBetaZone: zone,
+			},
+		}
+	}
+	volumeName, err := e2epv.CreatePDWithRetryAndZone(ctx, zone)
+	framework.ExpectNoError(err)
+	return &azureDiskVolume{
+		volumeName: volumeName,
+	}
+}
+
+func (v *azureDiskVolume) DeleteVolume(ctx context.Context) {
+	_ = e2epv.DeletePDWithRetry(ctx, v.volumeName)
+}
+
 // AWS
 type awsDriver struct {
 	driverInfo storageframework.DriverInfo
@@ -1755,3 +1900,11 @@ func (v *azureFileVolume) DeleteVolume(ctx context.Context) {
 	err := e2epv.DeleteShare(v.accountName, v.shareName)
 	framework.ExpectNoError(err)
 }
+
+func (a *azureDiskDriver) GetTimeouts() *framework.TimeoutContext {
+	timeouts := framework.NewTimeoutContext()
+	timeouts.PodStart = time.Minute * 15
+	timeouts.PodDelete = time.Minute * 15
+	timeouts.PVDelete = time.Minute * 20
+	return timeouts
+}

test/e2e/storage/in_tree_volumes.go

@@ -38,6 +38,7 @@ var testDrivers = []func() storageframework.TestDriver{
 	drivers.InitEmptydirDriver,
 	drivers.InitCinderDriver,
 	drivers.InitVSphereDriver,
+	drivers.InitAzureDiskDriver,
 	drivers.InitAzureFileDriver,
 	drivers.InitAwsDriver,
 	drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectory),

vendor/github.com/google/cadvisor/utils/cloudinfo/azure/azure.go

@@ -0,0 +1,58 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudinfo
+
+import (
+	"io/ioutil"
+	"strings"
+
+	info "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/utils/cloudinfo"
+)
+
+const (
+	sysVendorFileName    = "/sys/class/dmi/id/sys_vendor"
+	biosUUIDFileName     = "/sys/class/dmi/id/product_uuid"
+	microsoftCorporation = "Microsoft Corporation"
+)
+
+func init() {
+	cloudinfo.RegisterCloudProvider(info.Azure, &provider{})
+}
+
+type provider struct{}
+
+var _ cloudinfo.CloudProvider = provider{}
+
+func (provider) IsActiveProvider() bool {
+	data, err := ioutil.ReadFile(sysVendorFileName)
+	if err != nil {
+		return false
+	}
+	return strings.Contains(string(data), microsoftCorporation)
+}
+
+// TODO: Implement method.
+func (provider) GetInstanceType() info.InstanceType {
+	return info.UnknownInstance
+}
+
+func (provider) GetInstanceID() info.InstanceID {
+	data, err := ioutil.ReadFile(biosUUIDFileName)
+	if err != nil {
+		return info.UnNamedInstance
+	}
+	return info.InstanceID(strings.TrimSuffix(string(data), "\n"))
+}

vendor/modules.txt

@@ -384,6 +384,7 @@ github.com/google/cadvisor/third_party/containerd/api/types
 github.com/google/cadvisor/third_party/containerd/api/types/task
 github.com/google/cadvisor/utils
 github.com/google/cadvisor/utils/cloudinfo
+github.com/google/cadvisor/utils/cloudinfo/azure
 github.com/google/cadvisor/utils/cloudinfo/gce
 github.com/google/cadvisor/utils/cpuload
 github.com/google/cadvisor/utils/cpuload/netlink