From d5c3a959f43e056e70cf7731604f242dbf3b1f1c Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Mon, 19 Nov 2018 16:15:42 +0800 Subject: [PATCH] Add new Azure cloud provider option CloudProviderBackoffMode --- pkg/cloudprovider/providers/azure/azure.go | 105 +++++++++++++-------- 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/pkg/cloudprovider/providers/azure/azure.go b/pkg/cloudprovider/providers/azure/azure.go index a64e5730c42..d834935d0b0 100644 --- a/pkg/cloudprovider/providers/azure/azure.go +++ b/pkg/cloudprovider/providers/azure/azure.go @@ -59,6 +59,9 @@ const ( vmTypeVMSS = "vmss" vmTypeStandard = "standard" + backoffModeDefault = "default" + backoffModeV2 = "v2" + loadBalancerSkuBasic = "basic" loadBalancerSkuStandard = "standard" @@ -115,6 +118,12 @@ type Config struct { CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration" yaml:"cloudProviderBackoffDuration"` // Backoff jitter CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter" yaml:"cloudProviderBackoffJitter"` + // Backoff mode, options are v2 and default. + // * default means two-layer backoff retrying, one in the cloud provider and the other in the Azure SDK. + // * v2 means only backoff in the Azure SDK is used. In such mode, CloudProviderBackoffDuration and + // CloudProviderBackoffJitter are omitted. + // "default" will be used if not specified. + CloudProviderBackoffMode string `json:"cloudProviderBackoffMode" yaml:"cloudProviderBackoffMode"` // Enable rate limiting CloudProviderRateLimit bool `json:"cloudProviderRateLimit" yaml:"cloudProviderRateLimit"` // Rate limit QPS (Read) @@ -268,25 +277,71 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) { config.CloudProviderRateLimitBucketWrite) } + // Conditionally configure resource request backoff + resourceRequestBackoff := wait.Backoff{ + Steps: 1, + } + if config.CloudProviderBackoff { + // Assign backoff defaults if no configuration was passed in + if config.CloudProviderBackoffRetries == 0 { + config.CloudProviderBackoffRetries = backoffRetriesDefault + } + if config.CloudProviderBackoffDuration == 0 { + config.CloudProviderBackoffDuration = backoffDurationDefault + } + if config.CloudProviderBackoffExponent == 0 { + config.CloudProviderBackoffExponent = backoffExponentDefault + } else if config.shouldOmitCloudProviderBackoff() { + klog.Warning("Azure cloud provider config 'cloudProviderBackoffExponent' has been deprecated for 'v2' backoff mode. 2 is always used as the backoff exponent.") + } + if config.CloudProviderBackoffJitter == 0 { + config.CloudProviderBackoffJitter = backoffJitterDefault + } else if config.shouldOmitCloudProviderBackoff() { + klog.Warning("Azure cloud provider config 'cloudProviderBackoffJitter' has been deprecated for 'v2' backoff mode.") + } + + if !config.shouldOmitCloudProviderBackoff() { + resourceRequestBackoff = wait.Backoff{ + Steps: config.CloudProviderBackoffRetries, + Factor: config.CloudProviderBackoffExponent, + Duration: time.Duration(config.CloudProviderBackoffDuration) * time.Second, + Jitter: config.CloudProviderBackoffJitter, + } + } + klog.V(2).Infof("Azure cloudprovider using try backoff: retries=%d, exponent=%f, duration=%d, jitter=%f", + config.CloudProviderBackoffRetries, + config.CloudProviderBackoffExponent, + config.CloudProviderBackoffDuration, + config.CloudProviderBackoffJitter) + } else { + // CloudProviderBackoffRetries will be set to 1 by default as the requirements of Azure SDK. + config.CloudProviderBackoffRetries = 1 + config.CloudProviderBackoffDuration = backoffDurationDefault + } + // Do not add master nodes to standard LB by default. if config.ExcludeMasterFromStandardLB == nil { config.ExcludeMasterFromStandardLB = &defaultExcludeMasterFromStandardLB } azClientConfig := &azClientConfig{ - subscriptionID: config.SubscriptionID, - resourceManagerEndpoint: env.ResourceManagerEndpoint, - servicePrincipalToken: servicePrincipalToken, - rateLimiterReader: operationPollRateLimiter, - rateLimiterWriter: operationPollRateLimiterWrite, + subscriptionID: config.SubscriptionID, + resourceManagerEndpoint: env.ResourceManagerEndpoint, + servicePrincipalToken: servicePrincipalToken, + rateLimiterReader: operationPollRateLimiter, + rateLimiterWriter: operationPollRateLimiterWrite, + CloudProviderBackoffRetries: config.CloudProviderBackoffRetries, + CloudProviderBackoffDuration: config.CloudProviderBackoffDuration, + ShouldOmitCloudProviderBackoff: config.shouldOmitCloudProviderBackoff(), } az := Cloud{ - Config: *config, - Environment: *env, - nodeZones: map[string]sets.String{}, - nodeResourceGroups: map[string]string{}, - unmanagedNodes: sets.NewString(), - routeCIDRs: map[string]string{}, + Config: *config, + Environment: *env, + nodeZones: map[string]sets.String{}, + nodeResourceGroups: map[string]string{}, + unmanagedNodes: sets.NewString(), + routeCIDRs: map[string]string{}, + resourceRequestBackoff: resourceRequestBackoff, DisksClient: newAzDisksClient(azClientConfig), RoutesClient: newAzRoutesClient(azClientConfig), @@ -304,34 +359,6 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) { FileClient: &azureFileClient{env: *env}, } - // Conditionally configure resource request backoff - if az.CloudProviderBackoff { - // Assign backoff defaults if no configuration was passed in - if az.CloudProviderBackoffRetries == 0 { - az.CloudProviderBackoffRetries = backoffRetriesDefault - } - if az.CloudProviderBackoffExponent == 0 { - az.CloudProviderBackoffExponent = backoffExponentDefault - } - if az.CloudProviderBackoffDuration == 0 { - az.CloudProviderBackoffDuration = backoffDurationDefault - } - if az.CloudProviderBackoffJitter == 0 { - az.CloudProviderBackoffJitter = backoffJitterDefault - } - az.resourceRequestBackoff = wait.Backoff{ - Steps: az.CloudProviderBackoffRetries, - Factor: az.CloudProviderBackoffExponent, - Duration: time.Duration(az.CloudProviderBackoffDuration) * time.Second, - Jitter: az.CloudProviderBackoffJitter, - } - klog.V(2).Infof("Azure cloudprovider using try backoff: retries=%d, exponent=%f, duration=%d, jitter=%f", - az.CloudProviderBackoffRetries, - az.CloudProviderBackoffExponent, - az.CloudProviderBackoffDuration, - az.CloudProviderBackoffJitter) - } - az.metadata, err = NewInstanceMetadataService(metadataURL) if err != nil { return nil, err