Merge pull request #84605 from andyzhangx/byok

add azure disk encryption(SSE+CMK) support

pkg/volume/azure_dd/azure_provision.go

@@ -131,8 +131,9 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
 		availabilityZones        sets.String
 		selectedAvailabilityZone string
 
-		diskIopsReadWrite string
+		diskIopsReadWrite   string
-		diskMbpsReadWrite string
+		diskMbpsReadWrite   string
+		diskEncryptionSetID string
 	)
 	// maxLength = 79 - (4 for ".vhd") = 75
 	name := util.GenerateVolumeName(p.options.ClusterName, p.options.PVName, 75)
@@ -175,6 +176,8 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
 			diskIopsReadWrite = v
 		case "diskmbpsreadwrite":
 			diskMbpsReadWrite = v
+		case "diskencryptionsetid":
+			diskEncryptionSetID = v
 		default:
 			return nil, fmt.Errorf("AzureDisk - invalid option %s in storage class", k)
 		}
@@ -244,15 +247,16 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
 		}
 
 		volumeOptions := &azure.ManagedDiskOptions{
-			DiskName:           name,
+			DiskName:            name,
-			StorageAccountType: skuName,
+			StorageAccountType:  skuName,
-			ResourceGroup:      resourceGroup,
+			ResourceGroup:       resourceGroup,
-			PVCName:            p.options.PVC.Name,
+			PVCName:             p.options.PVC.Name,
-			SizeGB:             requestGiB,
+			SizeGB:              requestGiB,
-			Tags:               tags,
+			Tags:                tags,
-			AvailabilityZone:   selectedAvailabilityZone,
+			AvailabilityZone:    selectedAvailabilityZone,
-			DiskIOPSReadWrite:  diskIopsReadWrite,
+			DiskIOPSReadWrite:   diskIopsReadWrite,
-			DiskMBpsReadWrite:  diskMbpsReadWrite,
+			DiskMBpsReadWrite:   diskMbpsReadWrite,
+			DiskEncryptionSetID: diskEncryptionSetID,
 		}
 		diskURI, err = diskController.CreateManagedDisk(volumeOptions)
 		if err != nil {

staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common.go

@@ -98,6 +98,7 @@ func (c *controllerCommon) getNodeVMSet(nodeName types.NodeName, crt cacheReadTy
 // AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI.
 // return (lun, error)
 func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, cachingMode compute.CachingTypes) (int32, error) {
+	diskEncryptionSetID := ""
 	if isManagedDisk {
 		diskName := path.Base(diskURI)
 		resourceGroup, err := getResourceGroupFromDiskURI(diskURI)
@@ -122,6 +123,11 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
 			danglingErr := volerr.NewDanglingError(attachErr, types.NodeName(attachedNode), "")
 			return -1, danglingErr
 		}
+
+		if disk.DiskProperties != nil && disk.DiskProperties.Encryption != nil &&
+			disk.DiskProperties.Encryption.DiskEncryptionSetID != nil {
+			diskEncryptionSetID = *disk.DiskProperties.Encryption.DiskEncryptionSetID
+		}
 	}
 
 	vmset, err := c.getNodeVMSet(nodeName, cacheReadTypeUnsafe)
@@ -145,7 +151,7 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
 	}
 
 	klog.V(2).Infof("Trying to attach volume %q lun %d to node %q.", diskURI, lun, nodeName)
-	return lun, vmset.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode)
+	return lun, vmset.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode, diskEncryptionSetID)
 }
 
 // DetachDisk detaches a disk from host. The vhd can be identified by diskName or diskURI.

staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard.go

@@ -30,7 +30,7 @@ import (
 
 // AttachDisk attaches a vhd to vm
 // the vhd must exist, can be identified by diskName, diskURI, and lun.
-func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
+func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes, diskEncryptionSetID string) error {
 	vm, err := as.getVirtualMachine(nodeName, cacheReadTypeDefault)
 	if err != nil {
 		return err
@@ -46,15 +46,17 @@ func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI stri
 	copy(disks, *vm.StorageProfile.DataDisks)
 
 	if isManagedDisk {
+		managedDisk := &compute.ManagedDiskParameters{ID: &diskURI}
+		if diskEncryptionSetID != "" {
+			managedDisk.DiskEncryptionSet = &compute.DiskEncryptionSetParameters{ID: &diskEncryptionSetID}
+		}
 		disks = append(disks,
 			compute.DataDisk{
 				Name:         &diskName,
 				Lun:          &lun,
 				Caching:      cachingMode,
 				CreateOption: "attach",
-				ManagedDisk: &compute.ManagedDiskParameters{
+				ManagedDisk:  managedDisk,
-					ID: &diskURI,
-				},
 			})
 	} else {
 		disks = append(disks,
@@ -77,7 +79,7 @@ func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI stri
 			},
 		},
 	}
-	klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s)", nodeResourceGroup, vmName, diskName, diskURI)
+	klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s) with DiskEncryptionSetID(%s)", nodeResourceGroup, vmName, diskName, diskURI, diskEncryptionSetID)
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 

staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard_test.go

@@ -53,7 +53,7 @@ func TestStandardAttachDisk(t *testing.T) {
 		setTestVirtualMachines(testCloud, map[string]string{"vm1": "PowerState/Running"}, false)
 
 		err := vmSet.AttachDisk(true, "",
-			"uri", test.nodeName, 0, compute.CachingTypesReadOnly)
+			"uri", test.nodeName, 0, compute.CachingTypesReadOnly, "")
 		assert.Equal(t, test.expectedErr, err != nil, "TestCase[%d]: %s", i, test.desc)
 	}
 }

staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss.go

@@ -30,7 +30,7 @@ import (
 
 // AttachDisk attaches a vhd to vm
 // the vhd must exist, can be identified by diskName, diskURI, and lun.
-func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
+func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes, diskEncryptionSetID string) error {
 	vmName := mapNodeNameToVMName(nodeName)
 	ssName, instanceID, vm, err := ss.getVmssVM(vmName, cacheReadTypeDefault)
 	if err != nil {
@@ -48,15 +48,17 @@ func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nod
 		copy(disks, *vm.StorageProfile.DataDisks)
 	}
 	if isManagedDisk {
+		managedDisk := &compute.ManagedDiskParameters{ID: &diskURI}
+		if diskEncryptionSetID != "" {
+			managedDisk.DiskEncryptionSet = &compute.DiskEncryptionSetParameters{ID: &diskEncryptionSetID}
+		}
 		disks = append(disks,
 			compute.DataDisk{
 				Name:         &diskName,
 				Lun:          &lun,
 				Caching:      compute.CachingTypes(cachingMode),
 				CreateOption: "attach",
-				ManagedDisk: &compute.ManagedDiskParameters{
+				ManagedDisk:  managedDisk,
-					ID: &diskURI,
-				},
 			})
 	} else {
 		disks = append(disks,
@@ -90,7 +92,7 @@ func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nod
 		return err
 	}
 
-	klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s)", nodeResourceGroup, nodeName, diskName, diskURI)
+	klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s) with DiskEncryptionSetID(%s)", nodeResourceGroup, nodeName, diskName, diskURI, diskEncryptionSetID)
 	_, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM, "attach_disk")
 	if err != nil {
 		detail := err.Error()

staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go

@@ -942,7 +942,7 @@ func (f *fakeVMSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID,
 	return fmt.Errorf("unimplemented")
 }
 
-func (f *fakeVMSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
+func (f *fakeVMSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes, diskEncryptionSetID string) error {
 	return fmt.Errorf("unimplemented")
 }
 

staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go

@@ -40,6 +40,8 @@ const (
 	// default IOPS Caps & Throughput Cap (MBps) per https://docs.microsoft.com/en-us/azure/virtual-machines/linux/disks-ultra-ssd
 	defaultDiskIOPSReadWrite = 500
 	defaultDiskMBpsReadWrite = 100
+
+	diskEncryptionSetIDFormat = "/subscriptions/{subs-id}/resourceGroups/{rg-name}/providers/Microsoft.Compute/diskEncryptionSets/{diskEncryptionSet-name}"
 )
 
 //ManagedDiskController : managed disk controller struct
@@ -67,6 +69,8 @@ type ManagedDiskOptions struct {
 	DiskIOPSReadWrite string
 	// Throughput Cap (MBps) for UltraSSD disk
 	DiskMBpsReadWrite string
+	// ResourceId of the disk encryption set to use for enabling encryption at rest.
+	DiskEncryptionSetID string
 }
 
 //CreateManagedDisk : create managed disk
@@ -129,6 +133,16 @@ func (c *ManagedDiskController) CreateManagedDisk(options *ManagedDiskOptions) (
 		}
 	}
 
+	if options.DiskEncryptionSetID != "" {
+		if strings.Index(strings.ToLower(options.DiskEncryptionSetID), "/subscriptions/") != 0 {
+			return "", fmt.Errorf("AzureDisk - format of DiskEncryptionSetID(%s) is incorrect, correct format: %s", options.DiskEncryptionSetID, diskEncryptionSetIDFormat)
+		}
+		diskProperties.Encryption = &compute.Encryption{
+			DiskEncryptionSetID: &options.DiskEncryptionSetID,
+			Type:                compute.EncryptionAtRestWithCustomerKey,
+		}
+	}
+
 	model := compute.Disk{
 		Location: &c.common.location,
 		Tags:     newTags,

staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go

@@ -66,7 +66,7 @@ type VMSet interface {
 	EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error
 
 	// AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI, and lun.
-	AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error
+	AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes, diskEncryptionSetID string) error
 	// DetachDisk detaches a vhd from host. The vhd can be identified by diskName or diskURI.
 	DetachDisk(diskName, diskURI string, nodeName types.NodeName) (*http.Response, error)
 	// GetDataDisks gets a list of data disks attached to the node.