From c1383e99af59939038d36cf8fe68071f989adb30 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Fri, 14 Feb 2020 18:21:29 +0800 Subject: [PATCH] Add deletion interfaces for VM, VMSS and interface clients --- .../azure/azure_fakes.go | 12 ++++ .../interfaceclient/azure_interfaceclient.go | 43 ++++++++++++ .../azure_interfaceclient_test.go | 13 ++++ .../clients/interfaceclient/interface.go | 3 + .../mockinterfaceclient/interface.go | 14 ++++ .../azure/clients/vmclient/azure_vmclient.go | 43 ++++++++++++ .../clients/vmclient/azure_vmclient_test.go | 13 ++++ .../azure/clients/vmclient/interface.go | 3 + .../vmclient/mockvmclient/interface.go | 14 ++++ .../clients/vmssclient/azure_vmssclient.go | 66 +++++++++++++++++++ .../vmssclient/azure_vmssclient_test.go | 23 +++++++ .../azure/clients/vmssclient/interface.go | 3 + .../vmssclient/mockvmssclient/interface.go | 14 ++++ 13 files changed, 264 insertions(+) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go index 27965b4972a..e5287ae1fed 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go @@ -292,6 +292,10 @@ func (fIC *fakeAzureInterfacesClient) GetVirtualMachineScaleSetNetworkInterface( errors.New("Not such Interface")) } +func (fIC *fakeAzureInterfacesClient) Delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) *retry.Error { + return nil +} + func (fIC *fakeAzureInterfacesClient) setFakeStore(store map[string]map[string]network.Interface) { fIC.mutex.Lock() defer fIC.mutex.Unlock() @@ -363,6 +367,10 @@ func (fVMC *fakeAzureVirtualMachinesClient) List(ctx context.Context, resourceGr return result, nil } +func (fVMC *fakeAzureVirtualMachinesClient) Delete(ctx context.Context, resourceGroupName string, VMName string) *retry.Error { + return nil +} + func (fVMC *fakeAzureVirtualMachinesClient) setFakeStore(store map[string]map[string]compute.VirtualMachine) { fVMC.mutex.Lock() defer fVMC.mutex.Unlock() @@ -661,6 +669,10 @@ func (fVMSSC *fakeVirtualMachineScaleSetsClient) UpdateInstances(ctx context.Con return nil } +func (fVMSSC *fakeVirtualMachineScaleSetsClient) DeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) *retry.Error { + return nil +} + type fakeRoutesClient struct { mutex *sync.Mutex FakeStore map[string]map[string]network.Route diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient.go index f71cd2b3b9a..656c9106611 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient.go @@ -274,3 +274,46 @@ func (c *Client) createOrUpdateResponder(resp *http.Response) (*network.Interfac result.Response = autorest.Response{Response: resp} return result, retry.GetError(resp, err) } + +// Delete deletes a network interface by name. +func (c *Client) Delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) *retry.Error { + mc := metrics.NewMetricContext("interfaces", "delete", resourceGroupName, c.subscriptionID, "") + + // Report errors if the client is rate limited. + if !c.rateLimiterWriter.TryAccept() { + mc.RateLimitedCount() + return retry.GetRateLimitError(true, "NicDelete") + } + + // Report errors if the client is throttled. + if c.RetryAfterWriter.After(time.Now()) { + mc.ThrottledCount() + rerr := retry.GetThrottlingError("NicDelete", "client throttled", c.RetryAfterWriter) + return rerr + } + + rerr := c.deleteInterface(ctx, resourceGroupName, networkInterfaceName) + mc.Observe(rerr.Error()) + if rerr != nil { + if rerr.IsThrottled() { + // Update RetryAfterReader so that no more requests would be sent until RetryAfter expires. + c.RetryAfterWriter = rerr.RetryAfter + } + + return rerr + } + + return nil +} + +// deleteInterface deletes a network interface by name. +func (c *Client) deleteInterface(ctx context.Context, resourceGroupName string, networkInterfaceName string) *retry.Error { + resourceID := armclient.GetResourceID( + c.subscriptionID, + resourceGroupName, + "Microsoft.Network/networkInterfaces", + networkInterfaceName, + ) + + return c.armClient.DeleteResource(ctx, resourceID, "") +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient_test.go index 50ddb039369..269ef1b9f1e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/azure_interfaceclient_test.go @@ -121,6 +121,19 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestDelete(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + r := getTestInterface("interface1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().DeleteResource(gomock.Any(), to.String(r.ID), "").Return(nil).Times(1) + + diskClient := getTestInterfaceClient(armClient) + rerr := diskClient.Delete(context.TODO(), "rg", "interface1") + assert.Nil(t, rerr) +} + func getTestInterface(name string) network.Interface { resourceID := fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/%s", name) return network.Interface{ diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/interface.go index ca107a4bd33..0ea631481bf 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/interface.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/interface.go @@ -45,4 +45,7 @@ type Interface interface { // CreateOrUpdate creates or updates a network.Interface. CreateOrUpdate(ctx context.Context, resourceGroupName string, networkInterfaceName string, parameters network.Interface) *retry.Error + + // Delete deletes a network interface by name. + Delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) *retry.Error } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient/interface.go index 71c025f0c65..8050fd634ab 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient/interface.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient/interface.go @@ -93,3 +93,17 @@ func (mr *MockInterfaceMockRecorder) CreateOrUpdate(ctx, resourceGroupName, netw mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockInterface)(nil).CreateOrUpdate), ctx, resourceGroupName, networkInterfaceName, parameters) } + +// Delete mocks base method +func (m *MockInterface) Delete(ctx context.Context, resourceGroupName, networkInterfaceName string) *retry.Error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, resourceGroupName, networkInterfaceName) + ret0, _ := ret[0].(*retry.Error) + return ret0 +} + +// Delete indicates an expected call of Delete +func (mr *MockInterfaceMockRecorder) Delete(ctx, resourceGroupName, networkInterfaceName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockInterface)(nil).Delete), ctx, resourceGroupName, networkInterfaceName) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient.go index 54c26a38c68..932e3992582 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient.go @@ -434,3 +434,46 @@ func (c *Client) createOrUpdateResponder(resp *http.Response) (*compute.VirtualM result.Response = autorest.Response{Response: resp} return result, retry.GetError(resp, err) } + +// Delete deletes a VirtualMachine. +func (c *Client) Delete(ctx context.Context, resourceGroupName string, VMName string) *retry.Error { + mc := metrics.NewMetricContext("vm", "delete", resourceGroupName, c.subscriptionID, "") + + // Report errors if the client is rate limited. + if !c.rateLimiterWriter.TryAccept() { + mc.RateLimitedCount() + return retry.GetRateLimitError(true, "VMDelete") + } + + // Report errors if the client is throttled. + if c.RetryAfterWriter.After(time.Now()) { + mc.ThrottledCount() + rerr := retry.GetThrottlingError("VMDelete", "client throttled", c.RetryAfterWriter) + return rerr + } + + rerr := c.deleteVM(ctx, resourceGroupName, VMName) + mc.Observe(rerr.Error()) + if rerr != nil { + if rerr.IsThrottled() { + // Update RetryAfterReader so that no more requests would be sent until RetryAfter expires. + c.RetryAfterWriter = rerr.RetryAfter + } + + return rerr + } + + return nil +} + +// deleteVM deletes a VirtualMachine. +func (c *Client) deleteVM(ctx context.Context, resourceGroupName string, VMName string) *retry.Error { + resourceID := armclient.GetResourceID( + c.subscriptionID, + resourceGroupName, + "Microsoft.Compute/virtualMachines", + VMName, + ) + + return c.armClient.DeleteResource(ctx, resourceID, "") +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go index f28313c6a18..d72e4c3be5d 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go @@ -139,6 +139,19 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestDelete(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + r := getTestVM("vm1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().DeleteResource(gomock.Any(), to.String(r.ID), "").Return(nil).Times(1) + + client := getTestVMClient(armClient) + rerr := client.Delete(context.TODO(), "rg", "vm1") + assert.Nil(t, rerr) +} + func getTestVM(vmName string) compute.VirtualMachine { resourceID := fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/%s", vmName) return compute.VirtualMachine{ diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/interface.go index 23eabc88d6c..ad63e8b515e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/interface.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/interface.go @@ -45,4 +45,7 @@ type Interface interface { // Update updates a VirtualMachine. Update(ctx context.Context, resourceGroupName string, VMName string, parameters compute.VirtualMachineUpdate, source string) *retry.Error + + // Delete deletes a VirtualMachine. + Delete(ctx context.Context, resourceGroupName string, VMName string) *retry.Error } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/interface.go index 44e34359c20..1ba612bd23e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/interface.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/interface.go @@ -107,3 +107,17 @@ func (mr *MockInterfaceMockRecorder) Update(ctx, resourceGroupName, VMName, para mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockInterface)(nil).Update), ctx, resourceGroupName, VMName, parameters, source) } + +// Delete mocks base method +func (m *MockInterface) Delete(ctx context.Context, resourceGroupName, VMName string) *retry.Error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, resourceGroupName, VMName) + ret0, _ := ret[0].(*retry.Error) + return ret0 +} + +// Delete indicates an expected call of Delete +func (mr *MockInterfaceMockRecorder) Delete(ctx, resourceGroupName, VMName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockInterface)(nil).Delete), ctx, resourceGroupName, VMName) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient.go index ae2fe62b8a7..bf69bb50153 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient.go @@ -359,3 +359,69 @@ func (page VirtualMachineScaleSetListResultPage) Values() []compute.VirtualMachi } return *page.vmsslr.Value } + +// DeleteInstances deletes the instances for a VirtualMachineScaleSet. +func (c *Client) DeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) *retry.Error { + mc := metrics.NewMetricContext("vmss", "delete_instances", resourceGroupName, c.subscriptionID, "") + + // Report errors if the client is rate limited. + if !c.rateLimiterWriter.TryAccept() { + mc.RateLimitedCount() + return retry.GetRateLimitError(true, "VMSSDeleteInstances") + } + + // Report errors if the client is throttled. + if c.RetryAfterWriter.After(time.Now()) { + mc.ThrottledCount() + rerr := retry.GetThrottlingError("VMSSDeleteInstances", "client throttled", c.RetryAfterWriter) + return rerr + } + + rerr := c.deleteVMSSInstances(ctx, resourceGroupName, vmScaleSetName, vmInstanceIDs) + mc.Observe(rerr.Error()) + if rerr != nil { + if rerr.IsThrottled() { + // Update RetryAfterReader so that no more requests would be sent until RetryAfter expires. + c.RetryAfterWriter = rerr.RetryAfter + } + + return rerr + } + + return nil +} + +// deleteVMSSInstances deletes the instances for a VirtualMachineScaleSet. +func (c *Client) deleteVMSSInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) *retry.Error { + resourceID := armclient.GetResourceID( + c.subscriptionID, + resourceGroupName, + "Microsoft.Compute/virtualMachineScaleSets", + vmScaleSetName, + ) + response, rerr := c.armClient.PostResource(ctx, resourceID, "delete", vmInstanceIDs) + defer c.armClient.CloseResponse(ctx, response) + if rerr != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vmss.deletevms.request", resourceID, rerr.Error()) + return rerr + } + + err := autorest.Respond(response, azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusAccepted)) + if err != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vmss.deletevms.respond", resourceID, rerr.Error()) + return retry.GetError(response, err) + } + + future, err := azure.NewFutureFromResponse(response) + if err != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vmss.deletevms.future", resourceID, rerr.Error()) + return retry.NewError(false, err) + } + + if err := c.armClient.WaitForAsyncOperationCompletion(ctx, &future, "vmssclient.DeleteInstances"); err != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vmss.deletevms.wait", resourceID, rerr.Error()) + return retry.NewError(false, err) + } + + return nil +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient_test.go index 3ee19cdb85d..1997a49498d 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient_test.go @@ -119,6 +119,29 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestDeleteInstances(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + r := getTestVMSS("vmss1") + vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ + InstanceIds: &[]string{"0", "1", "2"}, + } + response := &http.Response{ + StatusCode: http.StatusOK, + Request: &http.Request{Method: "POST"}, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PostResource(gomock.Any(), to.String(r.ID), "delete", vmInstanceIDs).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + armClient.EXPECT().WaitForAsyncOperationCompletion(gomock.Any(), gomock.Any(), "vmssclient.DeleteInstances").Return(nil).Times(1) + + client := getTestVMSSClient(armClient) + rerr := client.DeleteInstances(context.TODO(), "rg", "vmss1", vmInstanceIDs) + assert.Nil(t, rerr) +} + func getTestVMSS(name string) compute.VirtualMachineScaleSet { return compute.VirtualMachineScaleSet{ ID: to.StringPtr("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1"), diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/interface.go index 91a7b2a3fd4..545cb5f2a99 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/interface.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/interface.go @@ -42,4 +42,7 @@ type Interface interface { // CreateOrUpdate creates or updates a VirtualMachineScaleSet. CreateOrUpdate(ctx context.Context, resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet) *retry.Error + + // DeleteInstances deletes the instances for a VirtualMachineScaleSet. + DeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) *retry.Error } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient/interface.go index c80827e1eba..60fe4666b7a 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient/interface.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient/interface.go @@ -93,3 +93,17 @@ func (mr *MockInterfaceMockRecorder) CreateOrUpdate(ctx, resourceGroupName, VMSc mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockInterface)(nil).CreateOrUpdate), ctx, resourceGroupName, VMScaleSetName, parameters) } + +// DeleteInstances mocks base method +func (m *MockInterface) DeleteInstances(ctx context.Context, resourceGroupName, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) *retry.Error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteInstances", ctx, resourceGroupName, vmScaleSetName, vmInstanceIDs) + ret0, _ := ret[0].(*retry.Error) + return ret0 +} + +// DeleteInstances indicates an expected call of DeleteInstances +func (mr *MockInterfaceMockRecorder) DeleteInstances(ctx, resourceGroupName, vmScaleSetName, vmInstanceIDs interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteInstances", reflect.TypeOf((*MockInterface)(nil).DeleteInstances), ctx, resourceGroupName, vmScaleSetName, vmInstanceIDs) +}