diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure.go index 4b875aef409..988f9af9df9 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure.go @@ -74,6 +74,17 @@ const ( managedByAzureLabel = "kubernetes.azure.com/managed" ) +const ( + // PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured + PreConfiguredBackendPoolLoadBalancerTypesNone = "" + // PreConfiguredBackendPoolLoadBalancerTypesInteral means that the `internal` load balancers are pre-configured + PreConfiguredBackendPoolLoadBalancerTypesInteral = "internal" + // PreConfiguredBackendPoolLoadBalancerTypesExternal means that the `external` load balancers are pre-configured + PreConfiguredBackendPoolLoadBalancerTypesExternal = "external" + // PreConfiguredBackendPoolLoadBalancerTypesAll means that all load balancers are pre-configured + PreConfiguredBackendPoolLoadBalancerTypesAll = "all" +) + var ( // Master nodes are not added to standard load balancer by default. defaultExcludeMasterFromStandardLB = true @@ -174,6 +185,13 @@ type Config struct { // LoadBalancerResourceGroup determines the specific resource group of the load balancer user want to use, working // with LoadBalancerName LoadBalancerResourceGroup string `json:"loadBalancerResourceGroup,omitempty" yaml:"loadBalancerResourceGroup,omitempty"` + // PreConfiguredBackendPoolLoadBalancerTypes determines whether the LoadBalancer BackendPool has been preconfigured. + // Candidate values are: + // "": exactly with today (not pre-configured for any LBs) + // "internal": for internal LoadBalancer + // "external": for external LoadBalancer + // "all": for both internal and external LoadBalancer + PreConfiguredBackendPoolLoadBalancerTypes string `json:"preConfiguredBackendPoolLoadBalancerTypes,omitempty" yaml:"preConfiguredBackendPoolLoadBalancerTypes,omitempty"` // AvailabilitySetNodesCacheTTLInSeconds sets the Cache TTL for availabilitySetNodesCache // if not set, will use default value diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go index 401fefab828..2705530fef7 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go @@ -684,6 +684,7 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend // nodes only used if wantLb is true func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node, wantLb bool) (*network.LoadBalancer, error) { isInternal := requiresInternalLoadBalancer(service) + isBackendPoolPreConfigured := az.isBackendPoolPreConfigured(service) serviceName := getServiceName(service) klog.V(2).Infof("reconcileLoadBalancer for service(%s) - wantLb(%t): started", serviceName, wantLb) lb, _, _, err := az.getServiceLoadBalancer(service, clusterName, nodes, wantLb) @@ -723,6 +724,14 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, } } if !foundBackendPool { + if isBackendPoolPreConfigured { + klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - PreConfiguredBackendPoolLoadBalancerTypes %s has been set but can not find corresponding backend pool, ignoring it", + serviceName, + wantLb, + az.PreConfiguredBackendPoolLoadBalancerTypes) + isBackendPoolPreConfigured = false + } + newBackendPools = append(newBackendPools, network.BackendAddressPool{ Name: to.StringPtr(lbBackendPoolName), }) @@ -928,28 +937,32 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, // If it is not exist, and no change to that, we don't CreateOrUpdate LB if dirtyLb { if lb.FrontendIPConfigurations == nil || len(*lb.FrontendIPConfigurations) == 0 { - // When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself, - // because an Azure load balancer cannot have an empty FrontendIPConfigurations collection - klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName) + if isBackendPoolPreConfigured { + klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - ignore cleanup of dirty lb because the lb is pre-confiruged", serviceName, lbName) + } else { + // When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself, + // because an Azure load balancer cannot have an empty FrontendIPConfigurations collection + klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName) - // Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB. - vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) - klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName) - err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools) - if err != nil { - klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err) - return nil, err - } - klog.V(10).Infof("EnsureBackendPoolDeleted(%s) for service %s: end", lbBackendPoolID, serviceName) + // Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB. + vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) + klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName) + err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools) + if err != nil { + klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err) + return nil, err + } + klog.V(10).Infof("EnsureBackendPoolDeleted(%s) for service %s: end", lbBackendPoolID, serviceName) - // Remove the LB. - klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLB(%q): start", lbName) - err = az.DeleteLB(service, lbName) - if err != nil { - klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName) - return nil, err + // Remove the LB. + klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLB(%q): start", lbName) + err = az.DeleteLB(service, lbName) + if err != nil { + klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName) + return nil, err + } + klog.V(10).Infof("az.DeleteLB(%q): end", lbName) } - klog.V(10).Infof("az.DeleteLB(%q): end", lbName) } else { klog.V(2).Infof("reconcileLoadBalancer: reconcileLoadBalancer for service(%s): lb(%s) - updating", serviceName, lbName) err := az.CreateOrUpdateLB(service, *lb) @@ -973,7 +986,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, } } - if wantLb && nodes != nil { + if wantLb && nodes != nil && !isBackendPoolPreConfigured { // Add the machines to the backend pool if they're not already vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) // Etag would be changed when updating backend pools, so invalidate lbCache after it. @@ -1695,6 +1708,23 @@ func (az *Cloud) getPublicIPAddressResourceGroup(service *v1.Service) string { return az.ResourceGroup } +func (az *Cloud) isBackendPoolPreConfigured(service *v1.Service) bool { + preConfigured := false + isInternal := requiresInternalLoadBalancer(service) + + if az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesAll { + preConfigured = true + } + if (az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesInteral) && isInternal { + preConfigured = true + } + if (az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesExternal) && !isInternal { + preConfigured = true + } + + return preConfigured +} + // Check if service requires an internal load balancer. func requiresInternalLoadBalancer(service *v1.Service) bool { if l, found := service.Annotations[ServiceAnnotationLoadBalancerInternal]; found { diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go index c4eec3507d7..eee9c57a571 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go @@ -1421,6 +1421,7 @@ func TestReconcileLoadBalancer(t *testing.T) { desc string service v1.Service loadBalancerSku string + preConfigLBType string disableOutboundSnat *bool wantLb bool existingLB network.LoadBalancer @@ -1456,6 +1457,16 @@ func TestReconcileLoadBalancer(t *testing.T) { expectedLB: expectedLb1, expectedError: nil, }, + { + desc: "reconcileLoadBalancer shall not raise an error", + loadBalancerSku: "basic", + service: service3, + existingLB: modifiedLb1, + preConfigLBType: "external", + wantLb: true, + expectedLB: expectedLb1, + expectedError: nil, + }, { desc: "reconcileLoadBalancer shall remove and reconstruct the correspoind field of lb and set enableTcpReset to false in lbRule", loadBalancerSku: "standard", @@ -1500,6 +1511,9 @@ func TestReconcileLoadBalancer(t *testing.T) { az := getTestCloud() az.Config.LoadBalancerSku = test.loadBalancerSku az.DisableOutboundSNAT = test.disableOutboundSnat + if test.preConfigLBType != "" { + az.Config.PreConfiguredBackendPoolLoadBalancerTypes = test.preConfigLBType + } clusterResources := getClusterResources(az, 3, 3) test.service.Spec.LoadBalancerIP = "1.2.3.4" @@ -2064,3 +2078,75 @@ func TestShouldUpdateLoadBalancer(t *testing.T) { assert.Equal(t, test.expectedOutput, shouldUpdateLoadBalancer, "TestCase[%d]: %s", i, test.desc) } } + +func TestIsBackendPoolPreConfigured(t *testing.T) { + testCases := []struct { + desc string + preConfiguredBackendPoolLoadBalancerTypes string + isInternalService bool + expectedOutput bool + }{ + { + desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is both for any case", + preConfiguredBackendPoolLoadBalancerTypes: "all", + isInternalService: true, + expectedOutput: true, + }, + { + desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is both for any case", + preConfiguredBackendPoolLoadBalancerTypes: "all", + isInternalService: false, + expectedOutput: true, + }, + { + desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is external when creating external lb", + preConfiguredBackendPoolLoadBalancerTypes: "external", + isInternalService: false, + expectedOutput: true, + }, + { + desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is external when creating internal lb", + preConfiguredBackendPoolLoadBalancerTypes: "external", + isInternalService: true, + expectedOutput: false, + }, + { + desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is internal when creating external lb", + preConfiguredBackendPoolLoadBalancerTypes: "internal", + isInternalService: false, + expectedOutput: false, + }, + { + desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is internal when creating internal lb", + preConfiguredBackendPoolLoadBalancerTypes: "internal", + isInternalService: true, + expectedOutput: true, + }, + { + desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is empty for any case", + preConfiguredBackendPoolLoadBalancerTypes: "", + isInternalService: true, + expectedOutput: false, + }, + { + desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is empty for any case", + preConfiguredBackendPoolLoadBalancerTypes: "", + isInternalService: false, + expectedOutput: false, + }, + } + + for i, test := range testCases { + az := getTestCloud() + az.Config.PreConfiguredBackendPoolLoadBalancerTypes = test.preConfiguredBackendPoolLoadBalancerTypes + var service v1.Service + if test.isInternalService { + service = getInternalTestService("test", 80) + } else { + service = getTestService("test", v1.ProtocolTCP, nil, 80) + } + + isPreConfigured := az.isBackendPoolPreConfigured(&service) + assert.Equal(t, test.expectedOutput, isPreConfigured, "TestCase[%d]: %s", i, test.desc) + } +}