Merge pull request #86266 from zqingqing1/cache-ttl-configurable

Make cloud provider caching TTL configurable

staging/src/k8s.io/legacy-cloud-providers/azure/azure.go

@@ -174,6 +174,22 @@ type Config struct {
 	// LoadBalancerResourceGroup determines the specific resource group of the load balancer user want to use, working
 	// with LoadBalancerName
 	LoadBalancerResourceGroup string `json:"loadBalancerResourceGroup,omitempty" yaml:"loadBalancerResourceGroup,omitempty"`
+
+	// AvailabilitySetNodesCacheTTLInSeconds sets the Cache TTL for availabilitySetNodesCache
+	// if not set, will use default value
+	AvailabilitySetNodesCacheTTLInSeconds int `json:"availabilitySetNodesCacheTTLInSeconds,omitempty" yaml:"availabilitySetNodesCacheTTLInSeconds,omitempty"`
+	// VmssCacheTTLInSeconds sets the cache TTL for VMSS
+	VmssCacheTTLInSeconds int `json:"vmssCacheTTLInSeconds,omitempty" yaml:"vmssCacheTTLInSeconds,omitempty"`
+	// VmssVirtualMachinesCacheTTLInSeconds sets the cache TTL for vmssVirtualMachines
+	VmssVirtualMachinesCacheTTLInSeconds int `json:"vmssVirtualMachinesCacheTTLInSeconds,omitempty" yaml:"vmssVirtualMachinesCacheTTLInSeconds,omitempty"`
+	// VmCacheTTLInSeconds sets the cache TTL for vm
+	VMCacheTTLInSeconds int `json:"vmCacheTTLInSeconds,omitempty" yaml:"vmCacheTTLInSeconds,omitempty"`
+	// LoadBalancerCacheTTLInSeconds sets the cache TTL for load balancer
+	LoadBalancerCacheTTLInSeconds int `json:"loadBalancerCacheTTLInSeconds,omitempty" yaml:"loadBalancerCacheTTLInSeconds,omitempty"`
+	// NsgCacheTTLInSeconds sets the cache TTL for network security group
+	NsgCacheTTLInSeconds int `json:"nsgCacheTTLInSeconds,omitempty" yaml:"nsgCacheTTLInSeconds,omitempty"`
+	// RouteTableCacheTTLInSeconds sets the cache TTL for route table
+	RouteTableCacheTTLInSeconds int `json:"routeTableCacheTTLInSeconds,omitempty" yaml:"routeTableCacheTTLInSeconds,omitempty"`
 }
 
 var _ cloudprovider.Interface = (*Cloud)(nil)

staging/src/k8s.io/legacy-cloud-providers/azure/azure_test.go

@@ -59,6 +59,13 @@ func TestParseConfig(t *testing.T) {
 		"CloudProviderRateLimitBucketWrite": 1,
 		"cloudProviderRateLimitQPS": 1,
 		"CloudProviderRateLimitQPSWrite": 1,
+		"availabilitySetNodesCacheTTLInSeconds": 100,
+		"vmssCacheTTLInSeconds": 100,
+		"vmssVirtualMachinesCacheTTLInSeconds": 100,
+		"vmCacheTTLInSeconds": 100,
+		"loadBalancerCacheTTLInSeconds": 100,
+		"nsgCacheTTLInSeconds": 100,
+		"routeTableCacheTTLInSeconds": 100,
 		"location": "location",
 		"maximumLoadBalancerRuleCount": 1,
 		"primaryAvailabilitySetName": "primaryAvailabilitySetName",
@@ -87,29 +94,36 @@ func TestParseConfig(t *testing.T) {
 			TenantID:                    "tenantId",
 			UseManagedIdentityExtension: true,
 		},
-		CloudProviderBackoff:              true,
+		CloudProviderBackoff:                  true,
-		CloudProviderBackoffDuration:      1,
+		CloudProviderBackoffDuration:          1,
-		CloudProviderBackoffExponent:      1,
+		CloudProviderBackoffExponent:          1,
-		CloudProviderBackoffJitter:        1,
+		CloudProviderBackoffJitter:            1,
-		CloudProviderBackoffRetries:       1,
+		CloudProviderBackoffRetries:           1,
-		CloudProviderRateLimit:            true,
+		CloudProviderRateLimit:                true,
-		CloudProviderRateLimitBucket:      1,
+		CloudProviderRateLimitBucket:          1,
-		CloudProviderRateLimitBucketWrite: 1,
+		CloudProviderRateLimitBucketWrite:     1,
-		CloudProviderRateLimitQPS:         1,
+		CloudProviderRateLimitQPS:             1,
-		CloudProviderRateLimitQPSWrite:    1,
+		CloudProviderRateLimitQPSWrite:        1,
-		Location:                          "location",
+		AvailabilitySetNodesCacheTTLInSeconds: 100,
-		MaximumLoadBalancerRuleCount:      1,
+		VmssCacheTTLInSeconds:                 100,
-		PrimaryAvailabilitySetName:        "primaryAvailabilitySetName",
+		VmssVirtualMachinesCacheTTLInSeconds:  100,
-		PrimaryScaleSetName:               "primaryScaleSetName",
+		VMCacheTTLInSeconds:                   100,
-		ResourceGroup:                     "resourcegroup",
+		LoadBalancerCacheTTLInSeconds:         100,
-		RouteTableName:                    "routeTableName",
+		NsgCacheTTLInSeconds:                  100,
-		RouteTableResourceGroup:           "routeTableResourceGroup",
+		RouteTableCacheTTLInSeconds:           100,
-		SecurityGroupName:                 "securityGroupName",
+		Location:                              "location",
-		SubnetName:                        "subnetName",
+		MaximumLoadBalancerRuleCount:          1,
-		UseInstanceMetadata:               true,
+		PrimaryAvailabilitySetName:            "primaryAvailabilitySetName",
-		VMType:                            "standard",
+		PrimaryScaleSetName:                   "primaryScaleSetName",
-		VnetName:                          "vnetName",
+		ResourceGroup:                         "resourcegroup",
-		VnetResourceGroup:                 "vnetResourceGroup",
+		RouteTableName:                        "routeTableName",
+		RouteTableResourceGroup:               "routeTableResourceGroup",
+		SecurityGroupName:                     "securityGroupName",
+		SubnetName:                            "subnetName",
+		UseInstanceMetadata:                   true,
+		VMType:                                "standard",
+		VnetName:                              "vnetName",
+		VnetResourceGroup:                     "vnetResourceGroup",
 	}
 
 	buffer := bytes.NewBufferString(azureConfig)
@@ -1572,7 +1586,14 @@ func TestNewCloudFromJSON(t *testing.T) {
 		"cloudProviderBackoff": true,
 		"cloudProviderRatelimit": true,
 		"cloudProviderRateLimitQPS": 0.5,
-		"cloudProviderRateLimitBucket": 5
+		"cloudProviderRateLimitBucket": 5,
+		"availabilitySetNodesCacheTTLInSeconds": 100,
+		"vmssCacheTTLInSeconds": 100,
+		"vmssVirtualMachinesCacheTTLInSeconds": 100,
+		"vmCacheTTLInSeconds": 100,
+		"loadBalancerCacheTTLInSeconds": 100,
+		"nsgCacheTTLInSeconds": 100,
+		"routeTableCacheTTLInSeconds": 100,
 	}`
 	validateConfig(t, config)
 }
@@ -1622,6 +1643,13 @@ cloudProviderBackoffJitter: 1.0
 cloudProviderRatelimit: true
 cloudProviderRateLimitQPS: 0.5
 cloudProviderRateLimitBucket: 5
+availabilitySetNodesCacheTTLInSeconds: 100
+vmssCacheTTLInSeconds: 100
+vmssVirtualMachinesCacheTTLInSeconds: 100
+vmCacheTTLInSeconds: 100
+loadBalancerCacheTTLInSeconds: 100
+nsgCacheTTLInSeconds: 100
+routeTableCacheTTLInSeconds: 100
 `
 	validateConfig(t, config)
 }
@@ -1695,6 +1723,27 @@ func validateConfig(t *testing.T, config string) {
 	if azureCloud.CloudProviderRateLimitBucket != 5 {
 		t.Errorf("got incorrect value for CloudProviderRateLimitBucket")
 	}
+	if azureCloud.AvailabilitySetNodesCacheTTLInSeconds != 100 {
+		t.Errorf("got incorrect value for availabilitySetNodesCacheTTLInSeconds")
+	}
+	if azureCloud.VmssCacheTTLInSeconds != 100 {
+		t.Errorf("got incorrect value for vmssCacheTTLInSeconds")
+	}
+	if azureCloud.VmssVirtualMachinesCacheTTLInSeconds != 100 {
+		t.Errorf("got incorrect value for vmssVirtualMachinesCacheTTLInSeconds")
+	}
+	if azureCloud.VMCacheTTLInSeconds != 100 {
+		t.Errorf("got incorrect value for vmCacheTTLInSeconds")
+	}
+	if azureCloud.LoadBalancerCacheTTLInSeconds != 100 {
+		t.Errorf("got incorrect value for loadBalancerCacheTTLInSeconds")
+	}
+	if azureCloud.NsgCacheTTLInSeconds != 100 {
+		t.Errorf("got incorrect value for nsgCacheTTLInSeconds")
+	}
+	if azureCloud.RouteTableCacheTTLInSeconds != 100 {
+		t.Errorf("got incorrect value for routeTableCacheTTLInSeconds")
+	}
 }
 
 func getCloudFromConfig(t *testing.T, config string) *Cloud {

staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go

@@ -38,9 +38,9 @@ var (
 	vmssVirtualMachinesKey  = "k8svmssVirtualMachinesKey"
 	availabilitySetNodesKey = "k8sAvailabilitySetNodesKey"
 
-	availabilitySetNodesCacheTTL = 15 * time.Minute
+	availabilitySetNodesCacheTTLDefaultInSeconds = 900
-	vmssTTL                      = 10 * time.Minute
+	vmssCacheTTLDefaultInSeconds                 = 600
-	vmssVirtualMachinesTTL       = 10 * time.Minute
+	vmssVirtualMachinesCacheTTLDefaultInSeconds  = 600
 )
 
 type vmssVirtualMachinesEntry struct {
@@ -87,7 +87,10 @@ func (ss *scaleSet) newVMSSCache() (*timedCache, error) {
 		return localCache, nil
 	}
 
-	return newTimedcache(vmssTTL, getter)
+	if ss.Config.VmssCacheTTLInSeconds == 0 {
+		ss.Config.VmssCacheTTLInSeconds = vmssCacheTTLDefaultInSeconds
+	}
+	return newTimedcache(time.Duration(ss.Config.VmssCacheTTLInSeconds)*time.Second, getter)
 }
 
 func extractVmssVMName(name string) (string, string, error) {
@@ -147,7 +150,10 @@ func (ss *scaleSet) newVMSSVirtualMachinesCache() (*timedCache, error) {
 		return localCache, nil
 	}
 
-	return newTimedcache(vmssVirtualMachinesTTL, getter)
+	if ss.Config.VmssVirtualMachinesCacheTTLInSeconds == 0 {
+		ss.Config.VmssVirtualMachinesCacheTTLInSeconds = vmssVirtualMachinesCacheTTLDefaultInSeconds
+	}
+	return newTimedcache(time.Duration(ss.Config.VmssVirtualMachinesCacheTTLInSeconds)*time.Second, getter)
 }
 
 func (ss *scaleSet) deleteCacheForNode(nodeName string) error {
@@ -186,7 +192,10 @@ func (ss *scaleSet) newAvailabilitySetNodesCache() (*timedCache, error) {
 		return localCache, nil
 	}
 
-	return newTimedcache(availabilitySetNodesCacheTTL, getter)
+	if ss.Config.AvailabilitySetNodesCacheTTLInSeconds == 0 {
+		ss.Config.AvailabilitySetNodesCacheTTLInSeconds = availabilitySetNodesCacheTTLDefaultInSeconds
+	}
+	return newTimedcache(time.Duration(ss.Config.AvailabilitySetNodesCacheTTLInSeconds)*time.Second, getter)
 }
 
 func (ss *scaleSet) isNodeManagedByAvailabilitySet(nodeName string, crt cacheReadType) (bool, error) {

staging/src/k8s.io/legacy-cloud-providers/azure/azure_wrap.go

@@ -35,10 +35,10 @@ import (
 )
 
 var (
-	vmCacheTTL  = time.Minute
+	vmCacheTTLDefaultInSeconds           = 60
-	lbCacheTTL  = 2 * time.Minute
+	loadBalancerCacheTTLDefaultInSeconds = 120
-	nsgCacheTTL = 2 * time.Minute
+	nsgCacheTTLDefaultInSeconds          = 120
-	rtCacheTTL  = 2 * time.Minute
+	routeTableCacheTTLDefaultInSeconds   = 120
 
 	azureNodeProviderIDRE    = regexp.MustCompile(`^azure:///subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/(?:.*)`)
 	azureResourceGroupNameRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/(?:.*)`)
@@ -228,7 +228,10 @@ func (az *Cloud) newVMCache() (*timedCache, error) {
 		return &vm, nil
 	}
 
-	return newTimedcache(vmCacheTTL, getter)
+	if az.VMCacheTTLInSeconds == 0 {
+		az.VMCacheTTLInSeconds = vmCacheTTLDefaultInSeconds
+	}
+	return newTimedcache(time.Duration(az.VMCacheTTLInSeconds)*time.Second, getter)
 }
 
 func (az *Cloud) newLBCache() (*timedCache, error) {
@@ -250,7 +253,10 @@ func (az *Cloud) newLBCache() (*timedCache, error) {
 		return &lb, nil
 	}
 
-	return newTimedcache(lbCacheTTL, getter)
+	if az.LoadBalancerCacheTTLInSeconds == 0 {
+		az.LoadBalancerCacheTTLInSeconds = loadBalancerCacheTTLDefaultInSeconds
+	}
+	return newTimedcache(time.Duration(az.LoadBalancerCacheTTLInSeconds)*time.Second, getter)
 }
 
 func (az *Cloud) newNSGCache() (*timedCache, error) {
@@ -271,7 +277,10 @@ func (az *Cloud) newNSGCache() (*timedCache, error) {
 		return &nsg, nil
 	}
 
-	return newTimedcache(nsgCacheTTL, getter)
+	if az.NsgCacheTTLInSeconds == 0 {
+		az.NsgCacheTTLInSeconds = nsgCacheTTLDefaultInSeconds
+	}
+	return newTimedcache(time.Duration(az.NsgCacheTTLInSeconds)*time.Second, getter)
 }
 
 func (az *Cloud) newRouteTableCache() (*timedCache, error) {
@@ -292,7 +301,10 @@ func (az *Cloud) newRouteTableCache() (*timedCache, error) {
 		return &rt, nil
 	}
 
-	return newTimedcache(rtCacheTTL, getter)
+	if az.RouteTableCacheTTLInSeconds == 0 {
+		az.RouteTableCacheTTLInSeconds = routeTableCacheTTLDefaultInSeconds
+	}
+	return newTimedcache(time.Duration(az.RouteTableCacheTTLInSeconds)*time.Second, getter)
 }
 
 func (az *Cloud) useStandardLoadBalancer() bool {