From 2525ef9983dfc3fb1da5d64b3836ac15d98cdeaa Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 14 Jul 2017 15:16:47 -0700 Subject: [PATCH] VirtualMachinesClient.Get backoff in lb pool logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EnsureHostInPool() submits a GET to azure API for VM info. We’re seeing this on agent node kubelets and would like to enable configurable backoff engagement for 4xx responses to be able to slow down the rate of reconciliation, when appropriate. --- .../providers/azure/azure_backoff.go | 16 ++++++++++++++++ .../providers/azure/azure_loadbalancer.go | 13 ++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/providers/azure/azure_backoff.go b/pkg/cloudprovider/providers/azure/azure_backoff.go index 839592f3035..8d092983c13 100644 --- a/pkg/cloudprovider/providers/azure/azure_backoff.go +++ b/pkg/cloudprovider/providers/azure/azure_backoff.go @@ -43,6 +43,22 @@ func (az *Cloud) GetVirtualMachineWithRetry(name types.NodeName) (compute.Virtua return machine, exists, err } +// VirtualMachineClientGetWithRetry invokes az.VirtualMachinesClient.Get with exponential backoff retry +func (az *Cloud) VirtualMachineClientGetWithRetry(resourceGroup, vmName string, types compute.InstanceViewTypes) (compute.VirtualMachine, error) { + var machine compute.VirtualMachine + err := wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) { + var retryErr error + machine, retryErr = az.VirtualMachinesClient.Get(resourceGroup, vmName, types) + if retryErr != nil { + glog.Errorf("backoff: failure, will retry,err=%v", retryErr) + return false, nil + } + glog.V(2).Infof("backoff: success") + return true, nil + }) + return machine, err +} + // CreateOrUpdateSGWithRetry invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry func (az *Cloud) CreateOrUpdateSGWithRetry(sg network.SecurityGroup) error { return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) { diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go index 9b959b4e4cb..868cb611a73 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go @@ -25,6 +25,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" serviceapi "k8s.io/kubernetes/pkg/api/v1/service" + "github.com/Azure/azure-sdk-for-go/arm/compute" "github.com/Azure/azure-sdk-for-go/arm/network" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/glog" @@ -871,11 +872,21 @@ func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) b // This ensures the given VM's Primary NIC's Primary IP Configuration is // participating in the specified LoadBalancer Backend Pool. func (az *Cloud) ensureHostInPool(serviceName string, nodeName types.NodeName, backendPoolID string) error { + var machine compute.VirtualMachine vmName := mapNodeNameToVMName(nodeName) az.operationPollRateLimiter.Accept() machine, err := az.VirtualMachinesClient.Get(az.ResourceGroup, vmName, "") if err != nil { - return err + if az.CloudProviderBackoff { + glog.V(2).Infof("ensureHostInPool(%s, %s, %s) backing off", serviceName, nodeName, backendPoolID) + machine, err = az.VirtualMachineClientGetWithRetry(az.ResourceGroup, vmName, "") + if err != nil { + glog.V(2).Infof("ensureHostInPool(%s, %s, %s) abort backoff", serviceName, nodeName, backendPoolID) + return err + } + } else { + return err + } } primaryNicID, err := getPrimaryInterfaceID(machine)