diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go index 0a422946c9b..079b0ba7be3 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go @@ -185,6 +185,17 @@ func (c *Client) PreparePutRequest(ctx context.Context, decorators ...autorest.P return c.prepareRequest(ctx, decorators...) } +// PreparePatchRequest prepares patch request +func (c *Client) PreparePatchRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) { + decorators = append( + []autorest.PrepareDecorator{ + autorest.AsContentType("application/json; charset=utf-8"), + autorest.AsPatch(), + autorest.WithBaseURL(c.baseURI)}, + decorators...) + return c.prepareRequest(ctx, decorators...) +} + // PreparePostRequest prepares post request func (c *Client) PreparePostRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) { decorators = append( @@ -302,11 +313,15 @@ func (c *Client) GetResource(ctx context.Context, resourceID, expand string) (*h // PutResource puts a resource by resource ID func (c *Client) PutResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) { - decorators := []autorest.PrepareDecorator{ + putDecorators := []autorest.PrepareDecorator{ autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}), autorest.WithJSON(parameters), } + return c.PutResourceWithDecorators(ctx, resourceID, parameters, putDecorators) +} +// PutResourceWithDecorators puts a resource by resource ID +func (c *Client) PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) { request, err := c.PreparePutRequest(ctx, decorators...) if err != nil { klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.prepare", resourceID, err) @@ -340,6 +355,46 @@ func (c *Client) PutResource(ctx context.Context, resourceID string, parameters return response, nil } +// PatchResource patches a resource by resource ID +func (c *Client) PatchResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) { + decorators := []autorest.PrepareDecorator{ + autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}), + autorest.WithJSON(parameters), + } + + request, err := c.PreparePatchRequest(ctx, decorators...) + if err != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "patch.prepare", resourceID, err) + return nil, retry.NewError(false, err) + } + + future, resp, clientErr := c.SendAsync(ctx, request) + defer c.CloseResponse(ctx, resp) + if clientErr != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "patch.send", resourceID, clientErr.Error()) + return nil, clientErr + } + + response, err := c.WaitForAsyncOperationResult(ctx, future, "armclient.PatchResource") + if err != nil { + if response != nil { + klog.V(5).Infof("Received error in WaitForAsyncOperationResult: '%s', response code %d", err.Error(), response.StatusCode) + } else { + klog.V(5).Infof("Received error in WaitForAsyncOperationResult: '%s', no response", err.Error()) + } + + retriableErr := retry.GetError(response, err) + if !retriableErr.Retriable && + strings.Contains(strings.ToUpper(err.Error()), strings.ToUpper("InternalServerError")) { + klog.V(5).Infof("Received InternalServerError in WaitForAsyncOperationResult: '%s', setting error retriable", err.Error()) + retriableErr.Retriable = true + } + return nil, retriableErr + } + + return response, nil +} + // PutResourceAsync puts a resource by resource ID in async mode func (c *Client) PutResourceAsync(ctx context.Context, resourceID string, parameters interface{}) (*azure.Future, *retry.Error) { decorators := []autorest.PrepareDecorator{ diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/interface.go index 7ffeef4a89d..356cacd10fc 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/interface.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/interface.go @@ -61,6 +61,12 @@ type Interface interface { // PutResource puts a resource by resource ID PutResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) + // PutResourceWithDecorators puts a resource with decorators by resource ID + PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) + + // PatchResource patches a resource by resource ID + PatchResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) + // PutResourceAsync puts a resource by resource ID in async mode PutResourceAsync(ctx context.Context, resourceID string, parameters interface{}) (*azure.Future, *retry.Error) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient/interface.go index 724dcd6be21..4e13649b14a 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient/interface.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient/interface.go @@ -227,6 +227,36 @@ func (mr *MockInterfaceMockRecorder) PutResource(ctx, resourceID, parameters int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResource", reflect.TypeOf((*MockInterface)(nil).PutResource), ctx, resourceID, parameters) } +// PutResourceWithDecorators mocks base method +func (m *MockInterface) PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PutResourceWithDecorators", ctx, resourceID, parameters, decorators) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(*retry.Error) + return ret0, ret1 +} + +// PutResourceWithDecorators indicates an expected call of PutResourceWithDecorators +func (mr *MockInterfaceMockRecorder) PutResourceWithDecorators(ctx, resourceID, parameters, decorators interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourceWithDecorators", reflect.TypeOf((*MockInterface)(nil).PutResourceWithDecorators), ctx, resourceID, parameters, decorators) +} + +// PatchResource mocks base method +func (m *MockInterface) PatchResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PatchResource", ctx, resourceID, parameters) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(*retry.Error) + return ret0, ret1 +} + +// PatchResource indicates an expected call of PatchResource +func (mr *MockInterfaceMockRecorder) PatchResource(ctx, resourceID, parameters interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PatchResource", reflect.TypeOf((*MockInterface)(nil).PatchResource), ctx, resourceID, parameters) +} + // PutResourceAsync mocks base method func (m *MockInterface) PutResourceAsync(ctx context.Context, resourceID string, parameters interface{}) (*azure.Future, *retry.Error) { m.ctrl.T.Helper() diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/BUILD new file mode 100644 index 00000000000..dbdb1ad25ac --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/BUILD @@ -0,0 +1,58 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "azure_vmclient.go", + "doc.go", + "interface.go", + ], + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/azure/clients/vmclient", + importpath = "k8s.io/legacy-cloud-providers/azure/clients/vmclient", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/metrics:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library", + "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute:go_default_library", + "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", + "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library", + "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["azure_vmclient_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library", + "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute:go_default_library", + "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", + "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", + "//vendor/github.com/golang/mock/gomock:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient.go new file mode 100644 index 00000000000..54c26a38c68 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient.go @@ -0,0 +1,436 @@ +// +build !providerless + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmclient + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/to" + + "k8s.io/client-go/util/flowcontrol" + "k8s.io/klog" + azclients "k8s.io/legacy-cloud-providers/azure/clients" + "k8s.io/legacy-cloud-providers/azure/clients/armclient" + "k8s.io/legacy-cloud-providers/azure/metrics" + "k8s.io/legacy-cloud-providers/azure/retry" +) + +var _ Interface = &Client{} + +// Client implements VirtualMachine client Interface. +type Client struct { + armClient armclient.Interface + subscriptionID string + + // Rate limiting configures. + rateLimiterReader flowcontrol.RateLimiter + rateLimiterWriter flowcontrol.RateLimiter + + // ARM throttling configures. + RetryAfterReader time.Time + RetryAfterWriter time.Time +} + +// New creates a new VirtualMachine client with ratelimiting. +func New(config *azclients.ClientConfig) *Client { + baseURI := config.ResourceManagerEndpoint + authorizer := autorest.NewBearerAuthorizer(config.ServicePrincipalToken) + armClient := armclient.New(authorizer, baseURI, "", APIVersion, config.Location, config.Backoff) + rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig) + + klog.V(2).Infof("Azure VirtualMachine client (read ops) using rate limit config: QPS=%g, bucket=%d", + config.RateLimitConfig.CloudProviderRateLimitQPS, + config.RateLimitConfig.CloudProviderRateLimitBucket) + klog.V(2).Infof("Azure VirtualMachine client (write ops) using rate limit config: QPS=%g, bucket=%d", + config.RateLimitConfig.CloudProviderRateLimitQPSWrite, + config.RateLimitConfig.CloudProviderRateLimitBucketWrite) + + client := &Client{ + armClient: armClient, + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + subscriptionID: config.SubscriptionID, + } + + return client +} + +// Get gets a VirtualMachine. +func (c *Client) Get(ctx context.Context, resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (compute.VirtualMachine, *retry.Error) { + mc := metrics.NewMetricContext("vm", "get", resourceGroupName, c.subscriptionID, "") + + // Report errors if the client is rate limited. + if !c.rateLimiterReader.TryAccept() { + mc.RateLimitedCount() + return compute.VirtualMachine{}, retry.GetRateLimitError(false, "VMGet") + } + + // Report errors if the client is throttled. + if c.RetryAfterReader.After(time.Now()) { + mc.ThrottledCount() + rerr := retry.GetThrottlingError("VMGet", "client throttled", c.RetryAfterReader) + return compute.VirtualMachine{}, rerr + } + + result, rerr := c.getVM(ctx, resourceGroupName, VMName, expand) + mc.Observe(rerr.Error()) + if rerr != nil { + if rerr.IsThrottled() { + // Update RetryAfterReader so that no more requests would be sent until RetryAfter expires. + c.RetryAfterReader = rerr.RetryAfter + } + + return result, rerr + } + + return result, nil +} + +// getVM gets a VirtualMachine. +func (c *Client) getVM(ctx context.Context, resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (compute.VirtualMachine, *retry.Error) { + resourceID := armclient.GetResourceID( + c.subscriptionID, + resourceGroupName, + "Microsoft.Compute/virtualMachines", + VMName, + ) + result := compute.VirtualMachine{} + + response, rerr := c.armClient.GetResource(ctx, resourceID, string(expand)) + defer c.armClient.CloseResponse(ctx, response) + if rerr != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.get.request", resourceID, rerr.Error()) + return result, rerr + } + + err := autorest.Respond( + response, + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result)) + if err != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.get.respond", resourceID, err) + return result, retry.GetError(response, err) + } + + result.Response = autorest.Response{Response: response} + return result, nil +} + +// List gets a list of VirtualMachine in the resourceGroupName. +func (c *Client) List(ctx context.Context, resourceGroupName string) ([]compute.VirtualMachine, *retry.Error) { + mc := metrics.NewMetricContext("vm", "list", resourceGroupName, c.subscriptionID, "") + + // Report errors if the client is rate limited. + if !c.rateLimiterReader.TryAccept() { + mc.RateLimitedCount() + return nil, retry.GetRateLimitError(false, "VMList") + } + + // Report errors if the client is throttled. + if c.RetryAfterReader.After(time.Now()) { + mc.ThrottledCount() + rerr := retry.GetThrottlingError("VMList", "client throttled", c.RetryAfterReader) + return nil, rerr + } + + result, rerr := c.listVM(ctx, resourceGroupName) + mc.Observe(rerr.Error()) + if rerr != nil { + if rerr.IsThrottled() { + // Update RetryAfterReader so that no more requests would be sent until RetryAfter expires. + c.RetryAfterReader = rerr.RetryAfter + } + + return result, rerr + } + + return result, nil +} + +// listVM gets a list of VirtualMachines in the resourceGroupName. +func (c *Client) listVM(ctx context.Context, resourceGroupName string) ([]compute.VirtualMachine, *retry.Error) { + resourceID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines", + autorest.Encode("path", c.subscriptionID), + autorest.Encode("path", resourceGroupName), + ) + + result := make([]compute.VirtualMachine, 0) + page := &VirtualMachineListResultPage{} + page.fn = c.listNextResults + + resp, rerr := c.armClient.GetResource(ctx, resourceID, "") + defer c.armClient.CloseResponse(ctx, resp) + if rerr != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.list.request", resourceID, rerr.Error()) + return result, rerr + } + + var err error + page.vmlr, err = c.listResponder(resp) + if err != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.list.respond", resourceID, err) + return result, retry.GetError(resp, err) + } + + for page.NotDone() { + result = append(result, *page.Response().Value...) + if err = page.NextWithContext(ctx); err != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.list.next", resourceID, err) + return result, retry.GetError(page.Response().Response.Response, err) + } + } + + return result, nil +} + +// Update updates a VirtualMachine. +func (c *Client) Update(ctx context.Context, resourceGroupName string, VMName string, parameters compute.VirtualMachineUpdate, source string) *retry.Error { + mc := metrics.NewMetricContext("vm", "update", resourceGroupName, c.subscriptionID, source) + + // Report errors if the client is rate limited. + if !c.rateLimiterWriter.TryAccept() { + mc.RateLimitedCount() + return retry.GetRateLimitError(true, "VMUpdate") + } + + // Report errors if the client is throttled. + if c.RetryAfterWriter.After(time.Now()) { + mc.ThrottledCount() + rerr := retry.GetThrottlingError("VMUpdate", "client throttled", c.RetryAfterWriter) + return rerr + } + + rerr := c.updateVM(ctx, resourceGroupName, VMName, parameters, source) + mc.Observe(rerr.Error()) + if rerr != nil { + if rerr.IsThrottled() { + // Update RetryAfterReader so that no more requests would be sent until RetryAfter expires. + c.RetryAfterWriter = rerr.RetryAfter + } + + return rerr + } + + return nil +} + +// updateVM updates a VirtualMachine. +func (c *Client) updateVM(ctx context.Context, resourceGroupName string, VMName string, parameters compute.VirtualMachineUpdate, source string) *retry.Error { + resourceID := armclient.GetResourceID( + c.subscriptionID, + resourceGroupName, + "Microsoft.Compute/virtualMachines", + VMName, + ) + + response, rerr := c.armClient.PatchResource(ctx, resourceID, parameters) + defer c.armClient.CloseResponse(ctx, response) + if rerr != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.put.request", resourceID, rerr.Error()) + return rerr + } + + if response != nil && response.StatusCode != http.StatusNoContent { + _, rerr = c.updateResponder(response) + if rerr != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.put.respond", resourceID, rerr.Error()) + return rerr + } + } + + return nil +} + +func (c *Client) updateResponder(resp *http.Response) (*compute.VirtualMachine, *retry.Error) { + result := &compute.VirtualMachine{} + err := autorest.Respond( + resp, + azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusCreated), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return result, retry.GetError(resp, err) +} + +func (c *Client) listResponder(resp *http.Response) (result compute.VirtualMachineListResult, err error) { + err = autorest.Respond( + resp, + autorest.ByIgnoring(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing(), + ) + result.Response = autorest.Response{Response: resp} + return +} + +// vmListResultPreparer prepares a request to retrieve the next set of results. +// It returns nil if no more results exist. +func (c *Client) vmListResultPreparer(ctx context.Context, vmlr compute.VirtualMachineListResult) (*http.Request, error) { + if vmlr.NextLink == nil || len(to.String(vmlr.NextLink)) < 1 { + return nil, nil + } + + decorators := []autorest.PrepareDecorator{ + autorest.WithBaseURL(to.String(vmlr.NextLink)), + } + return c.armClient.PrepareGetRequest(ctx, decorators...) +} + +// listNextResults retrieves the next set of results, if any. +func (c *Client) listNextResults(ctx context.Context, lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error) { + req, err := c.vmListResultPreparer(ctx, lastResults) + if err != nil { + return result, autorest.NewErrorWithError(err, "vmclient", "listNextResults", nil, "Failure preparing next results request") + } + if req == nil { + return + } + + resp, rerr := c.armClient.Send(ctx, req) + defer c.armClient.CloseResponse(ctx, resp) + if rerr != nil { + result.Response = autorest.Response{Response: resp} + return result, autorest.NewErrorWithError(rerr.Error(), "vmclient", "listNextResults", resp, "Failure sending next results request") + } + + result, err = c.listResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "vmclient", "listNextResults", resp, "Failure responding to next results request") + } + + return +} + +// VirtualMachineListResultPage contains a page of VirtualMachine values. +type VirtualMachineListResultPage struct { + fn func(context.Context, compute.VirtualMachineListResult) (compute.VirtualMachineListResult, error) + vmlr compute.VirtualMachineListResult +} + +// NextWithContext advances to the next page of values. If there was an error making +// the request the page does not advance and the error is returned. +func (page *VirtualMachineListResultPage) NextWithContext(ctx context.Context) (err error) { + next, err := page.fn(ctx, page.vmlr) + if err != nil { + return err + } + page.vmlr = next + return nil +} + +// Next advances to the next page of values. If there was an error making +// the request the page does not advance and the error is returned. +// Deprecated: Use NextWithContext() instead. +func (page *VirtualMachineListResultPage) Next() error { + return page.NextWithContext(context.Background()) +} + +// NotDone returns true if the page enumeration should be started or is not yet complete. +func (page VirtualMachineListResultPage) NotDone() bool { + return !page.vmlr.IsEmpty() +} + +// Response returns the raw server response from the last page request. +func (page VirtualMachineListResultPage) Response() compute.VirtualMachineListResult { + return page.vmlr +} + +// Values returns the slice of values for the current page or nil if there are no values. +func (page VirtualMachineListResultPage) Values() []compute.VirtualMachine { + if page.vmlr.IsEmpty() { + return nil + } + return *page.vmlr.Value +} + +// CreateOrUpdate creates or updates a VirtualMachine. +func (c *Client) CreateOrUpdate(ctx context.Context, resourceGroupName string, VMName string, parameters compute.VirtualMachine, source string) *retry.Error { + mc := metrics.NewMetricContext("vm", "create_or_update", resourceGroupName, c.subscriptionID, source) + + // Report errors if the client is rate limited. + if !c.rateLimiterWriter.TryAccept() { + mc.RateLimitedCount() + return retry.GetRateLimitError(true, "VMCreateOrUpdate") + } + + // Report errors if the client is throttled. + if c.RetryAfterWriter.After(time.Now()) { + mc.ThrottledCount() + rerr := retry.GetThrottlingError("VMCreateOrUpdate", "client throttled", c.RetryAfterWriter) + return rerr + } + + rerr := c.createOrUpdateVM(ctx, resourceGroupName, VMName, parameters, source) + mc.Observe(rerr.Error()) + if rerr != nil { + if rerr.IsThrottled() { + // Update RetryAfterReader so that no more requests would be sent until RetryAfter expires. + c.RetryAfterWriter = rerr.RetryAfter + } + + return rerr + } + + return nil +} + +// createOrUpdateVM creates or updates a VirtualMachine. +func (c *Client) createOrUpdateVM(ctx context.Context, resourceGroupName string, VMName string, parameters compute.VirtualMachine, source string) *retry.Error { + resourceID := armclient.GetResourceID( + c.subscriptionID, + resourceGroupName, + "Microsoft.Compute/virtualMachines", + VMName, + ) + + response, rerr := c.armClient.PutResource(ctx, resourceID, parameters) + defer c.armClient.CloseResponse(ctx, response) + if rerr != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.put.request", resourceID, rerr.Error()) + return rerr + } + + if response != nil && response.StatusCode != http.StatusNoContent { + _, rerr = c.createOrUpdateResponder(response) + if rerr != nil { + klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "vm.put.respond", resourceID, rerr.Error()) + return rerr + } + } + + return nil +} + +func (c *Client) createOrUpdateResponder(resp *http.Response) (*compute.VirtualMachine, *retry.Error) { + result := &compute.VirtualMachine{} + err := autorest.Respond( + resp, + azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusCreated), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return result, retry.GetError(resp, err) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go new file mode 100644 index 00000000000..f28313c6a18 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go @@ -0,0 +1,159 @@ +// +build !providerless + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmclient + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "testing" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + + azclients "k8s.io/legacy-cloud-providers/azure/clients" + "k8s.io/legacy-cloud-providers/azure/clients/armclient" + "k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient" +) + +func TestGetNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + expectedVM := compute.VirtualMachine{Response: autorest.Response{}} + result, rerr := vmClient.Get(context.TODO(), "rg", "vm1", "InstanceView") + assert.Equal(t, expectedVM, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode) +} + +func TestGetInternalError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + response := &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + expectedVM := compute.VirtualMachine{Response: autorest.Response{}} + result, rerr := vmClient.Get(context.TODO(), "rg", "vm1", "InstanceView") + assert.Equal(t, expectedVM, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) +} + +func TestList(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines" + armClient := mockarmclient.NewMockInterface(ctrl) + vmList := []compute.VirtualMachine{getTestVM("vm1"), getTestVM("vm1"), getTestVM("vm1")} + responseBody, err := json.Marshal(compute.VirtualMachineListResult{Value: &vmList}) + assert.Nil(t, err) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return( + &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader(responseBody)), + }, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + result, rerr := vmClient.List(context.TODO(), "rg") + assert.Nil(t, rerr) + assert.Equal(t, 3, len(result)) +} + +func TestUpdate(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + testVM := compute.VirtualMachineUpdate{} + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + armClient.EXPECT().PatchResource(gomock.Any(), resourceID, testVM).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + rerr := vmClient.Update(context.TODO(), "rg", "vm1", testVM, "test") + assert.Nil(t, rerr) +} + +func TestCreateOrUpdate(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testVM := getTestVM("vm1") + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + armClient.EXPECT().PutResource(gomock.Any(), to.String(testVM.ID), testVM).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + rerr := vmClient.CreateOrUpdate(context.TODO(), "rg", "vm1", testVM, "test") + assert.Nil(t, rerr) +} + +func getTestVM(vmName string) compute.VirtualMachine { + resourceID := fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/%s", vmName) + return compute.VirtualMachine{ + ID: to.StringPtr(resourceID), + Name: to.StringPtr(vmName), + Location: to.StringPtr("eastus"), + } +} + +func getTestVMClient(armClient armclient.Interface) *Client { + rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(&azclients.RateLimitConfig{}) + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + } +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/doc.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/doc.go new file mode 100644 index 00000000000..a18d94f5695 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/doc.go @@ -0,0 +1,20 @@ +// +build !providerless + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package vmclient implements the client for VirtualMachines. +package vmclient // import "k8s.io/legacy-cloud-providers/azure/clients/vmclient" diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/interface.go new file mode 100644 index 00000000000..23eabc88d6c --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/interface.go @@ -0,0 +1,48 @@ +// +build !providerless + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmclient + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" + "k8s.io/legacy-cloud-providers/azure/retry" +) + +const ( + // APIVersion is the API version for VirtualMachine. + APIVersion = "2019-07-01" +) + +// Interface is the client interface for VirtualMachines. +// Don't forget to run the following command to generate the mock client: +// mockgen -source=$GOPATH/src/k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/interface.go -package=mockvmclient Interface > $GOPATH/src/k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/interface.go +type Interface interface { + // Get gets a VirtualMachine. + Get(ctx context.Context, resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (compute.VirtualMachine, *retry.Error) + + // List gets a list of VirtualMachines in the resourceGroupName. + List(ctx context.Context, resourceGroupName string) ([]compute.VirtualMachine, *retry.Error) + + // CreateOrUpdate creates or updates a VirtualMachine. + CreateOrUpdate(ctx context.Context, resourceGroupName string, VMName string, parameters compute.VirtualMachine, source string) *retry.Error + + // Update updates a VirtualMachine. + Update(ctx context.Context, resourceGroupName string, VMName string, parameters compute.VirtualMachineUpdate, source string) *retry.Error +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/BUILD new file mode 100644 index 00000000000..542249cb3ae --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "doc.go", + "interface.go", + ], + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient", + importpath = "k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library", + "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute:go_default_library", + "//vendor/github.com/golang/mock/gomock:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/doc.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/doc.go new file mode 100644 index 00000000000..19cd7052237 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/doc.go @@ -0,0 +1,20 @@ +// +build !providerless + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mockvmclient implements the mock client for VirtualMachines. +package mockvmclient // import "k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient" diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/interface.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/interface.go new file mode 100644 index 00000000000..44e34359c20 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient/interface.go @@ -0,0 +1,109 @@ +// +build !providerless + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mockvmclient + +import ( + context "context" + reflect "reflect" + + compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" + gomock "github.com/golang/mock/gomock" + retry "k8s.io/legacy-cloud-providers/azure/retry" +) + +// MockInterface is a mock of Interface interface +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// Get mocks base method +func (m *MockInterface) Get(ctx context.Context, resourceGroupName, VMName string, expand compute.InstanceViewTypes) (compute.VirtualMachine, *retry.Error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, VMName, expand) + ret0, _ := ret[0].(compute.VirtualMachine) + ret1, _ := ret[1].(*retry.Error) + return ret0, ret1 +} + +// Get indicates an expected call of Get +func (mr *MockInterfaceMockRecorder) Get(ctx, resourceGroupName, VMName, expand interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), ctx, resourceGroupName, VMName, expand) +} + +// List mocks base method +func (m *MockInterface) List(ctx context.Context, resourceGroupName string) ([]compute.VirtualMachine, *retry.Error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, resourceGroupName) + ret0, _ := ret[0].([]compute.VirtualMachine) + ret1, _ := ret[1].(*retry.Error) + return ret0, ret1 +} + +// List indicates an expected call of List +func (mr *MockInterfaceMockRecorder) List(ctx, resourceGroupName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockInterface)(nil).List), ctx, resourceGroupName) +} + +// CreateOrUpdate mocks base method +func (m *MockInterface) CreateOrUpdate(ctx context.Context, resourceGroupName, VMName string, parameters compute.VirtualMachine, source string) *retry.Error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrUpdate", ctx, resourceGroupName, VMName, parameters, source) + ret0, _ := ret[0].(*retry.Error) + return ret0 +} + +// CreateOrUpdate indicates an expected call of CreateOrUpdate +func (mr *MockInterfaceMockRecorder) CreateOrUpdate(ctx, resourceGroupName, VMName, parameters, source interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockInterface)(nil).CreateOrUpdate), ctx, resourceGroupName, VMName, parameters, source) +} + +// Update mocks base method +func (m *MockInterface) Update(ctx context.Context, resourceGroupName, VMName string, parameters compute.VirtualMachineUpdate, source string) *retry.Error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", ctx, resourceGroupName, VMName, parameters, source) + ret0, _ := ret[0].(*retry.Error) + return ret0 +} + +// Update indicates an expected call of Update +func (mr *MockInterfaceMockRecorder) Update(ctx, resourceGroupName, VMName, parameters, source interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockInterface)(nil).Update), ctx, resourceGroupName, VMName, parameters, source) +}