github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 15:05:27 +00:00
Code Issues Projects Releases Wiki Activity

Add a config option to azure cloud provider for the pre-configured loadbalancers

Browse Source
This commit is contained in:
guwe 2019-12-19 01:50:54 +00:00
parent 127c47caf4
commit 131180b118
3 changed files with 154 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
staging/src/k8s.io/legacy-cloud-providers/azure/azure.go
View File

@ -74,6 +74,17 @@ const (
managedByAzureLabel = "kubernetes.azure.com/managed"
)
const (
// PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured
PreConfiguredBackendPoolLoadBalancerTypesNone = ""
// PreConfiguredBackendPoolLoadBalancerTypesInteral means that the `internal` load balancers are pre-configured
PreConfiguredBackendPoolLoadBalancerTypesInteral = "internal"
// PreConfiguredBackendPoolLoadBalancerTypesExternal means that the `external` load balancers are pre-configured
PreConfiguredBackendPoolLoadBalancerTypesExternal = "external"
// PreConfiguredBackendPoolLoadBalancerTypesAll means that all load balancers are pre-configured
PreConfiguredBackendPoolLoadBalancerTypesAll = "all"
)
var (
// Master nodes are not added to standard load balancer by default.
defaultExcludeMasterFromStandardLB = true
@ -174,6 +185,13 @@ type Config struct {
// LoadBalancerResourceGroup determines the specific resource group of the load balancer user want to use, working
// with LoadBalancerName
LoadBalancerResourceGroup string `json:"loadBalancerResourceGroup,omitempty" yaml:"loadBalancerResourceGroup,omitempty"`
// PreConfiguredBackendPoolLoadBalancerTypes determines whether the LoadBalancer BackendPool has been preconfigured.
// Candidate values are:
// "": exactly with today (not pre-configured for any LBs)
// "internal": for internal LoadBalancer
// "external": for external LoadBalancer
// "all": for both internal and external LoadBalancer
PreConfiguredBackendPoolLoadBalancerTypes string `json:"preConfiguredBackendPoolLoadBalancerTypes,omitempty" yaml:"preConfiguredBackendPoolLoadBalancerTypes,omitempty"`
// AvailabilitySetNodesCacheTTLInSeconds sets the Cache TTL for availabilitySetNodesCache
// if not set, will use default value

70
staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
View File

@ -684,6 +684,7 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend
// nodes only used if wantLb is true
func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node, wantLb bool) (*network.LoadBalancer, error) {
isInternal := requiresInternalLoadBalancer(service)
isBackendPoolPreConfigured := az.isBackendPoolPreConfigured(service)
serviceName := getServiceName(service)
klog.V(2).Infof("reconcileLoadBalancer for service(%s) - wantLb(%t): started", serviceName, wantLb)
lb, _, _, err := az.getServiceLoadBalancer(service, clusterName, nodes, wantLb)
@ -723,6 +724,14 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
}
}
if !foundBackendPool {
if isBackendPoolPreConfigured {
klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - PreConfiguredBackendPoolLoadBalancerTypes %s has been set but can not find corresponding backend pool, ignoring it",
serviceName,
wantLb,
az.PreConfiguredBackendPoolLoadBalancerTypes)
isBackendPoolPreConfigured = false
}
newBackendPools = append(newBackendPools, network.BackendAddressPool{
Name: to.StringPtr(lbBackendPoolName),
})
@ -928,28 +937,32 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
// If it is not exist, and no change to that, we don't CreateOrUpdate LB
if dirtyLb {
if lb.FrontendIPConfigurations == nil || len(*lb.FrontendIPConfigurations) == 0 {
// When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself,
// because an Azure load balancer cannot have an empty FrontendIPConfigurations collection
klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
if isBackendPoolPreConfigured {
klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - ignore cleanup of dirty lb because the lb is pre-confiruged", serviceName, lbName)
} else {
// When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself,
// because an Azure load balancer cannot have an empty FrontendIPConfigurations collection
klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
if err != nil {
klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
return nil, err
}
klog.V(10).Infof("EnsureBackendPoolDeleted(%s) for service %s: end", lbBackendPoolID, serviceName)
// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
if err != nil {
klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
return nil, err
}
klog.V(10).Infof("EnsureBackendPoolDeleted(%s) for service %s: end", lbBackendPoolID, serviceName)
// Remove the LB.
klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLB(%q): start", lbName)
err = az.DeleteLB(service, lbName)
if err != nil {
klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
return nil, err
// Remove the LB.
klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLB(%q): start", lbName)
err = az.DeleteLB(service, lbName)
if err != nil {
klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
return nil, err
}
klog.V(10).Infof("az.DeleteLB(%q): end", lbName)
}
klog.V(10).Infof("az.DeleteLB(%q): end", lbName)
} else {
klog.V(2).Infof("reconcileLoadBalancer: reconcileLoadBalancer for service(%s): lb(%s) - updating", serviceName, lbName)
err := az.CreateOrUpdateLB(service, *lb)
@ -973,7 +986,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
}
}
if wantLb && nodes != nil {
if wantLb && nodes != nil && !isBackendPoolPreConfigured {
// Add the machines to the backend pool if they're not already
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
// Etag would be changed when updating backend pools, so invalidate lbCache after it.
@ -1695,6 +1708,23 @@ func (az *Cloud) getPublicIPAddressResourceGroup(service *v1.Service) string {
return az.ResourceGroup
}
func (az *Cloud) isBackendPoolPreConfigured(service *v1.Service) bool {
preConfigured := false
isInternal := requiresInternalLoadBalancer(service)
if az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesAll {
preConfigured = true
}
if (az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesInteral) && isInternal {
preConfigured = true
}
if (az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesExternal) && !isInternal {
preConfigured = true
}
return preConfigured
}
// Check if service requires an internal load balancer.
func requiresInternalLoadBalancer(service *v1.Service) bool {
if l, found := service.Annotations[ServiceAnnotationLoadBalancerInternal]; found {

86
staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go
View File

@ -1421,6 +1421,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
desc string
service v1.Service
loadBalancerSku string
preConfigLBType string
disableOutboundSnat *bool
wantLb bool
existingLB network.LoadBalancer
@ -1456,6 +1457,16 @@ func TestReconcileLoadBalancer(t *testing.T) {
expectedLB: expectedLb1,
expectedError: nil,
},
{
desc: "reconcileLoadBalancer shall not raise an error",
loadBalancerSku: "basic",
service: service3,
existingLB: modifiedLb1,
preConfigLBType: "external",
wantLb: true,
expectedLB: expectedLb1,
expectedError: nil,
},
{
desc: "reconcileLoadBalancer shall remove and reconstruct the correspoind field of lb and set enableTcpReset to false in lbRule",
loadBalancerSku: "standard",
@ -1500,6 +1511,9 @@ func TestReconcileLoadBalancer(t *testing.T) {
az := getTestCloud()
az.Config.LoadBalancerSku = test.loadBalancerSku
az.DisableOutboundSNAT = test.disableOutboundSnat
if test.preConfigLBType != "" {
az.Config.PreConfiguredBackendPoolLoadBalancerTypes = test.preConfigLBType
}
clusterResources := getClusterResources(az, 3, 3)
test.service.Spec.LoadBalancerIP = "1.2.3.4"
@ -2064,3 +2078,75 @@ func TestShouldUpdateLoadBalancer(t *testing.T) {
assert.Equal(t, test.expectedOutput, shouldUpdateLoadBalancer, "TestCase[%d]: %s", i, test.desc)
}
}
func TestIsBackendPoolPreConfigured(t *testing.T) {
testCases := []struct {
desc string
preConfiguredBackendPoolLoadBalancerTypes string
isInternalService bool
expectedOutput bool
}{
{
desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is both for any case",
preConfiguredBackendPoolLoadBalancerTypes: "all",
isInternalService: true,
expectedOutput: true,
},
{
desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is both for any case",
preConfiguredBackendPoolLoadBalancerTypes: "all",
isInternalService: false,
expectedOutput: true,
},
{
desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is external when creating external lb",
preConfiguredBackendPoolLoadBalancerTypes: "external",
isInternalService: false,
expectedOutput: true,
},
{
desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is external when creating internal lb",
preConfiguredBackendPoolLoadBalancerTypes: "external",
isInternalService: true,
expectedOutput: false,
},
{
desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is internal when creating external lb",
preConfiguredBackendPoolLoadBalancerTypes: "internal",
isInternalService: false,
expectedOutput: false,
},
{
desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is internal when creating internal lb",
preConfiguredBackendPoolLoadBalancerTypes: "internal",
isInternalService: true,
expectedOutput: true,
},
{
desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is empty for any case",
preConfiguredBackendPoolLoadBalancerTypes: "",
isInternalService: true,
expectedOutput: false,
},
{
desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is empty for any case",
preConfiguredBackendPoolLoadBalancerTypes: "",
isInternalService: false,
expectedOutput: false,
},
}
for i, test := range testCases {
az := getTestCloud()
az.Config.PreConfiguredBackendPoolLoadBalancerTypes = test.preConfiguredBackendPoolLoadBalancerTypes
var service v1.Service
if test.isInternalService {
service = getInternalTestService("test", 80)
} else {
service = getTestService("test", v1.ProtocolTCP, nil, 80)
}
isPreConfigured := az.isBackendPoolPreConfigured(&service)
assert.Equal(t, test.expectedOutput, isPreConfigured, "TestCase[%d]: %s", i, test.desc)
}
}