github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 20:24:09 +00:00
Code Issues Projects Releases Wiki Activity

VirtualMachinesClient.Get backoff in lb pool logic

Browse Source 
EnsureHostInPool() submits a GET to azure API for VM info. We’re seeing this on agent node kubelets and would like to enable configurable backoff engagement for 4xx responses to be able to slow down the rate of reconciliation, when appropriate.
This commit is contained in:
Jack Francis 2017-07-14 15:16:47 -07:00
parent 8532cdfd69
commit 2525ef9983
2 changed files with 28 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pkg/cloudprovider/providers/azure/azure_backoff.go
View File

@ -43,6 +43,22 @@ func (az *Cloud) GetVirtualMachineWithRetry(name types.NodeName) (compute.Virtua
return machine, exists, err
}
// VirtualMachineClientGetWithRetry invokes az.VirtualMachinesClient.Get with exponential backoff retry
func (az *Cloud) VirtualMachineClientGetWithRetry(resourceGroup, vmName string, types compute.InstanceViewTypes) (compute.VirtualMachine, error) {
var machine compute.VirtualMachine
err := wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
var retryErr error
machine, retryErr = az.VirtualMachinesClient.Get(resourceGroup, vmName, types)
if retryErr != nil {
glog.Errorf("backoff: failure, will retry,err=%v", retryErr)
return false, nil
}
glog.V(2).Infof("backoff: success")
return true, nil
})
return machine, err
}
// CreateOrUpdateSGWithRetry invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) CreateOrUpdateSGWithRetry(sg network.SecurityGroup) error {
return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {

13
pkg/cloudprovider/providers/azure/azure_loadbalancer.go
View File

@ -25,6 +25,7 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
serviceapi "k8s.io/kubernetes/pkg/api/v1/service"
"github.com/Azure/azure-sdk-for-go/arm/compute"
"github.com/Azure/azure-sdk-for-go/arm/network"
"github.com/Azure/go-autorest/autorest/to"
"github.com/golang/glog"
@ -871,11 +872,21 @@ func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) b
// This ensures the given VM's Primary NIC's Primary IP Configuration is
// participating in the specified LoadBalancer Backend Pool.
func (az *Cloud) ensureHostInPool(serviceName string, nodeName types.NodeName, backendPoolID string) error {
var machine compute.VirtualMachine
vmName := mapNodeNameToVMName(nodeName)
az.operationPollRateLimiter.Accept()
machine, err := az.VirtualMachinesClient.Get(az.ResourceGroup, vmName, "")
if err != nil {
return err
if az.CloudProviderBackoff {
glog.V(2).Infof("ensureHostInPool(%s, %s, %s) backing off", serviceName, nodeName, backendPoolID)
machine, err = az.VirtualMachineClientGetWithRetry(az.ResourceGroup, vmName, "")
if err != nil {
glog.V(2).Infof("ensureHostInPool(%s, %s, %s) abort backoff", serviceName, nodeName, backendPoolID)
return err
}
} else {
return err
}
}
primaryNicID, err := getPrimaryInterfaceID(machine)