From f10d44bad21078703aeefeb6d3435cd87d5d2830 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Thu, 31 Oct 2019 13:24:43 +0000 Subject: [PATCH 1/2] feat: add azure disk encryption(SSE+CMK) support --- pkg/volume/azure_dd/azure_provision.go | 26 +++++++++++-------- .../azure/azure_managedDiskController.go | 9 +++++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/pkg/volume/azure_dd/azure_provision.go b/pkg/volume/azure_dd/azure_provision.go index 1f6efb64a1e..6936e40428d 100644 --- a/pkg/volume/azure_dd/azure_provision.go +++ b/pkg/volume/azure_dd/azure_provision.go @@ -131,8 +131,9 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie availabilityZones sets.String selectedAvailabilityZone string - diskIopsReadWrite string - diskMbpsReadWrite string + diskIopsReadWrite string + diskMbpsReadWrite string + diskEncryptionSetID string ) // maxLength = 79 - (4 for ".vhd") = 75 name := util.GenerateVolumeName(p.options.ClusterName, p.options.PVName, 75) @@ -175,6 +176,8 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie diskIopsReadWrite = v case "diskmbpsreadwrite": diskMbpsReadWrite = v + case "diskencryptionsetid": + diskEncryptionSetID = v default: return nil, fmt.Errorf("AzureDisk - invalid option %s in storage class", k) } @@ -244,15 +247,16 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie } volumeOptions := &azure.ManagedDiskOptions{ - DiskName: name, - StorageAccountType: skuName, - ResourceGroup: resourceGroup, - PVCName: p.options.PVC.Name, - SizeGB: requestGiB, - Tags: tags, - AvailabilityZone: selectedAvailabilityZone, - DiskIOPSReadWrite: diskIopsReadWrite, - DiskMBpsReadWrite: diskMbpsReadWrite, + DiskName: name, + StorageAccountType: skuName, + ResourceGroup: resourceGroup, + PVCName: p.options.PVC.Name, + SizeGB: requestGiB, + Tags: tags, + AvailabilityZone: selectedAvailabilityZone, + DiskIOPSReadWrite: diskIopsReadWrite, + DiskMBpsReadWrite: diskMbpsReadWrite, + DiskEncryptionSetID: diskEncryptionSetID, } diskURI, err = diskController.CreateManagedDisk(volumeOptions) if err != nil { diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go index ebbae1da8d0..6417519c706 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go @@ -67,6 +67,8 @@ type ManagedDiskOptions struct { DiskIOPSReadWrite string // Throughput Cap (MBps) for UltraSSD disk DiskMBpsReadWrite string + // ResourceId of the disk encryption set to use for enabling encryption at rest. + DiskEncryptionSetID string } //CreateManagedDisk : create managed disk @@ -129,6 +131,13 @@ func (c *ManagedDiskController) CreateManagedDisk(options *ManagedDiskOptions) ( } } + if options.DiskEncryptionSetID != "" { + diskProperties.Encryption = &compute.Encryption{ + DiskEncryptionSetID: &options.DiskEncryptionSetID, + Type: compute.EncryptionAtRestWithCustomerKey, + } + } + model := compute.Disk{ Location: &c.common.location, Tags: newTags, From b26467b34446d6371bcdd0266a745f6d1c156ca0 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Fri, 1 Nov 2019 12:24:24 +0000 Subject: [PATCH 2/2] feat: add SSE+CMK support for azure disk add logging fix comment --- .../azure/azure_controller_common.go | 8 +++++++- .../azure/azure_controller_standard.go | 12 +++++++----- .../azure/azure_controller_standard_test.go | 2 +- .../azure/azure_controller_vmss.go | 12 +++++++----- .../legacy-cloud-providers/azure/azure_fakes.go | 2 +- .../azure/azure_managedDiskController.go | 5 +++++ .../legacy-cloud-providers/azure/azure_vmsets.go | 2 +- 7 files changed, 29 insertions(+), 14 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common.go index 9697950a349..20e1ca67200 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common.go @@ -98,6 +98,7 @@ func (c *controllerCommon) getNodeVMSet(nodeName types.NodeName, crt cacheReadTy // AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI. // return (lun, error) func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, cachingMode compute.CachingTypes) (int32, error) { + diskEncryptionSetID := "" if isManagedDisk { diskName := path.Base(diskURI) resourceGroup, err := getResourceGroupFromDiskURI(diskURI) @@ -122,6 +123,11 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri danglingErr := volerr.NewDanglingError(attachErr, types.NodeName(attachedNode), "") return -1, danglingErr } + + if disk.DiskProperties != nil && disk.DiskProperties.Encryption != nil && + disk.DiskProperties.Encryption.DiskEncryptionSetID != nil { + diskEncryptionSetID = *disk.DiskProperties.Encryption.DiskEncryptionSetID + } } vmset, err := c.getNodeVMSet(nodeName, cacheReadTypeUnsafe) @@ -145,7 +151,7 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri } klog.V(2).Infof("Trying to attach volume %q lun %d to node %q.", diskURI, lun, nodeName) - return lun, vmset.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode) + return lun, vmset.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode, diskEncryptionSetID) } // DetachDisk detaches a disk from host. The vhd can be identified by diskName or diskURI. diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard.go index 9b80280b102..995609d6899 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard.go @@ -30,7 +30,7 @@ import ( // AttachDisk attaches a vhd to vm // the vhd must exist, can be identified by diskName, diskURI, and lun. -func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error { +func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes, diskEncryptionSetID string) error { vm, err := as.getVirtualMachine(nodeName, cacheReadTypeDefault) if err != nil { return err @@ -46,15 +46,17 @@ func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI stri copy(disks, *vm.StorageProfile.DataDisks) if isManagedDisk { + managedDisk := &compute.ManagedDiskParameters{ID: &diskURI} + if diskEncryptionSetID != "" { + managedDisk.DiskEncryptionSet = &compute.DiskEncryptionSetParameters{ID: &diskEncryptionSetID} + } disks = append(disks, compute.DataDisk{ Name: &diskName, Lun: &lun, Caching: cachingMode, CreateOption: "attach", - ManagedDisk: &compute.ManagedDiskParameters{ - ID: &diskURI, - }, + ManagedDisk: managedDisk, }) } else { disks = append(disks, @@ -77,7 +79,7 @@ func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI stri }, }, } - klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s)", nodeResourceGroup, vmName, diskName, diskURI) + klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s) with DiskEncryptionSetID(%s)", nodeResourceGroup, vmName, diskName, diskURI, diskEncryptionSetID) ctx, cancel := getContextWithCancel() defer cancel() diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard_test.go index cd37631c623..bc78665c7a1 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_standard_test.go @@ -53,7 +53,7 @@ func TestStandardAttachDisk(t *testing.T) { setTestVirtualMachines(testCloud, map[string]string{"vm1": "PowerState/Running"}, false) err := vmSet.AttachDisk(true, "", - "uri", test.nodeName, 0, compute.CachingTypesReadOnly) + "uri", test.nodeName, 0, compute.CachingTypesReadOnly, "") assert.Equal(t, test.expectedErr, err != nil, "TestCase[%d]: %s", i, test.desc) } } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss.go index c54e63fcd67..2ca83e28aa1 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss.go @@ -30,7 +30,7 @@ import ( // AttachDisk attaches a vhd to vm // the vhd must exist, can be identified by diskName, diskURI, and lun. -func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error { +func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes, diskEncryptionSetID string) error { vmName := mapNodeNameToVMName(nodeName) ssName, instanceID, vm, err := ss.getVmssVM(vmName, cacheReadTypeDefault) if err != nil { @@ -48,15 +48,17 @@ func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nod copy(disks, *vm.StorageProfile.DataDisks) } if isManagedDisk { + managedDisk := &compute.ManagedDiskParameters{ID: &diskURI} + if diskEncryptionSetID != "" { + managedDisk.DiskEncryptionSet = &compute.DiskEncryptionSetParameters{ID: &diskEncryptionSetID} + } disks = append(disks, compute.DataDisk{ Name: &diskName, Lun: &lun, Caching: compute.CachingTypes(cachingMode), CreateOption: "attach", - ManagedDisk: &compute.ManagedDiskParameters{ - ID: &diskURI, - }, + ManagedDisk: managedDisk, }) } else { disks = append(disks, @@ -90,7 +92,7 @@ func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nod return err } - klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s)", nodeResourceGroup, nodeName, diskName, diskURI) + klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s, %s) with DiskEncryptionSetID(%s)", nodeResourceGroup, nodeName, diskName, diskURI, diskEncryptionSetID) _, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM, "attach_disk") if err != nil { detail := err.Error() diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go index 30799917835..8772cb7143c 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go @@ -942,7 +942,7 @@ func (f *fakeVMSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, return fmt.Errorf("unimplemented") } -func (f *fakeVMSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error { +func (f *fakeVMSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes, diskEncryptionSetID string) error { return fmt.Errorf("unimplemented") } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go index 6417519c706..847da0237c2 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go @@ -40,6 +40,8 @@ const ( // default IOPS Caps & Throughput Cap (MBps) per https://docs.microsoft.com/en-us/azure/virtual-machines/linux/disks-ultra-ssd defaultDiskIOPSReadWrite = 500 defaultDiskMBpsReadWrite = 100 + + diskEncryptionSetIDFormat = "/subscriptions/{subs-id}/resourceGroups/{rg-name}/providers/Microsoft.Compute/diskEncryptionSets/{diskEncryptionSet-name}" ) //ManagedDiskController : managed disk controller struct @@ -132,6 +134,9 @@ func (c *ManagedDiskController) CreateManagedDisk(options *ManagedDiskOptions) ( } if options.DiskEncryptionSetID != "" { + if strings.Index(strings.ToLower(options.DiskEncryptionSetID), "/subscriptions/") != 0 { + return "", fmt.Errorf("AzureDisk - format of DiskEncryptionSetID(%s) is incorrect, correct format: %s", options.DiskEncryptionSetID, diskEncryptionSetIDFormat) + } diskProperties.Encryption = &compute.Encryption{ DiskEncryptionSetID: &options.DiskEncryptionSetID, Type: compute.EncryptionAtRestWithCustomerKey, diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go index 2d7347ae113..c86ef86d6c1 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go @@ -66,7 +66,7 @@ type VMSet interface { EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error // AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI, and lun. - AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error + AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes, diskEncryptionSetID string) error // DetachDisk detaches a vhd from host. The vhd can be identified by diskName or diskURI. DetachDisk(diskName, diskURI string, nodeName types.NodeName) (*http.Response, error) // GetDataDisks gets a list of data disks attached to the node.