mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 11:21:47 +00:00
Merge pull request #95542 from nilo19/bug/support-exclude-node-from-lb-label
Support the node label `node.kubernetes.io/exclude-from-external-load-balancers`
This commit is contained in:
commit
d7e0cb0e35
@ -981,6 +981,15 @@ func (az *Cloud) findFrontendIPConfigOfService(
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
func nodeNameInNodes(nodeName string, nodes []*v1.Node) bool {
|
||||
for _, node := range nodes {
|
||||
if strings.EqualFold(nodeName, node.Name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// This ensures load balancer exists and the frontend ip config is setup.
|
||||
// This also reconciles the Service's Ports with the LoadBalancer config.
|
||||
// This entails adding rules/probes for expected Ports and removing stale rules/ports.
|
||||
@ -1022,6 +1031,42 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
|
||||
if strings.EqualFold(*bp.Name, lbBackendPoolName) {
|
||||
klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb)
|
||||
foundBackendPool = true
|
||||
|
||||
var backendIPConfigurationsToBeDeleted []network.InterfaceIPConfiguration
|
||||
if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil {
|
||||
for _, ipConf := range *bp.BackendIPConfigurations {
|
||||
ipConfID := to.String(ipConf.ID)
|
||||
nodeName, err := az.VMSet.GetNodeNameByIPConfigurationID(ipConfID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// If a node is not supposed to be included in the LB, it
|
||||
// would not be in the `nodes` slice. We need to check the nodes that
|
||||
// have been added to the LB's backendpool, find the unwanted ones and
|
||||
// delete them from the pool.
|
||||
if !nodeNameInNodes(nodeName, nodes) {
|
||||
klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found unwanted node %s, decouple it from the LB", serviceName, wantLb, nodeName)
|
||||
// construct a backendPool that only contains the IP config of the node to be deleted
|
||||
backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, network.InterfaceIPConfiguration{ID: to.StringPtr(ipConfID)})
|
||||
}
|
||||
}
|
||||
if len(backendIPConfigurationsToBeDeleted) > 0 {
|
||||
backendpoolToBeDeleted := &[]network.BackendAddressPool{
|
||||
{
|
||||
ID: to.StringPtr(lbBackendPoolID),
|
||||
BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
|
||||
BackendIPConfigurations: &backendIPConfigurationsToBeDeleted,
|
||||
},
|
||||
},
|
||||
}
|
||||
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
|
||||
// decouple the backendPool from the node
|
||||
err = az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, backendpoolToBeDeleted)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
break
|
||||
} else {
|
||||
klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found other backendpool %s", serviceName, wantLb, *bp.Name)
|
||||
@ -1294,6 +1339,10 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
|
||||
// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
|
||||
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
|
||||
klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
|
||||
if _, ok := az.VMSet.(*availabilitySet); ok {
|
||||
// do nothing for availability set
|
||||
lb.BackendAddressPools = nil
|
||||
}
|
||||
err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
|
||||
if err != nil {
|
||||
klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
|
||||
@ -2170,8 +2219,8 @@ func equalLoadBalancingRulePropertiesFormat(s *network.LoadBalancingRuleProperti
|
||||
reflect.DeepEqual(s.FrontendPort, t.FrontendPort) &&
|
||||
reflect.DeepEqual(s.BackendPort, t.BackendPort) &&
|
||||
reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP) &&
|
||||
reflect.DeepEqual(s.EnableTCPReset, t.EnableTCPReset) &&
|
||||
reflect.DeepEqual(s.DisableOutboundSnat, t.DisableOutboundSnat)
|
||||
reflect.DeepEqual(to.Bool(s.EnableTCPReset), to.Bool(t.EnableTCPReset)) &&
|
||||
reflect.DeepEqual(to.Bool(s.DisableOutboundSnat), to.Bool(t.DisableOutboundSnat))
|
||||
|
||||
if wantLB && s.IdleTimeoutInMinutes != nil && t.IdleTimeoutInMinutes != nil {
|
||||
return properties && reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes)
|
||||
|
@ -1762,11 +1762,12 @@ func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" +
|
||||
"Microsoft.Network/loadBalancers/" + *name + "/backendAddressPools/" + *clusterName),
|
||||
},
|
||||
LoadDistribution: network.LoadDistribution("Default"),
|
||||
FrontendPort: to.Int32Ptr(service.Spec.Ports[0].Port),
|
||||
BackendPort: to.Int32Ptr(service.Spec.Ports[0].Port),
|
||||
EnableFloatingIP: to.BoolPtr(true),
|
||||
EnableTCPReset: to.BoolPtr(strings.EqualFold(lbSku, "standard")),
|
||||
LoadDistribution: network.LoadDistribution("Default"),
|
||||
FrontendPort: to.Int32Ptr(service.Spec.Ports[0].Port),
|
||||
BackendPort: to.Int32Ptr(service.Spec.Ports[0].Port),
|
||||
EnableFloatingIP: to.BoolPtr(true),
|
||||
EnableTCPReset: to.BoolPtr(strings.EqualFold(lbSku, "standard")),
|
||||
DisableOutboundSnat: to.BoolPtr(false),
|
||||
Probe: &network.SubResource{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80"),
|
||||
},
|
||||
@ -1833,8 +1834,6 @@ func TestReconcileLoadBalancer(t *testing.T) {
|
||||
},
|
||||
}
|
||||
expectedLb1 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service3, "Basic")
|
||||
(*expectedLb1.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false)
|
||||
(*expectedLb1.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
|
||||
expectedLb1.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
|
||||
{
|
||||
Name: to.StringPtr("aservice1"),
|
||||
@ -1970,9 +1969,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
|
||||
lb6.Probes = &[]network.Probe{}
|
||||
expectedLB6 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service6, "basic")
|
||||
expectedLB6.Probes = &[]network.Probe{}
|
||||
(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].Probe = nil
|
||||
(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
|
||||
(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false)
|
||||
(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].Probe = &network.SubResource{ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80")}
|
||||
expectedLB6.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
|
||||
{
|
||||
Name: to.StringPtr("aservice1"),
|
||||
@ -1994,7 +1991,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-UDP-80"),
|
||||
}
|
||||
(*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
|
||||
(*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false)
|
||||
(*lb7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(true)
|
||||
expectedLB7.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
|
||||
{
|
||||
Name: to.StringPtr("aservice1"),
|
||||
|
@ -71,10 +71,14 @@ const (
|
||||
loadBalancerRuleNameMaxLength = 80
|
||||
)
|
||||
|
||||
var errNotInVMSet = errors.New("vm is not in the vmset")
|
||||
var providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
|
||||
var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`)
|
||||
var nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`)
|
||||
var (
|
||||
errNotInVMSet = errors.New("vm is not in the vmset")
|
||||
providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
|
||||
backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`)
|
||||
nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`)
|
||||
nicIDRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.+)/providers/Microsoft.Network/networkInterfaces/(.+)-nic-(.+)/ipConfigurations/(?:.*)`)
|
||||
vmasIDRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)`)
|
||||
)
|
||||
|
||||
// getStandardMachineID returns the full identifier of a virtual machine.
|
||||
func (az *Cloud) getStandardMachineID(subscriptionID, resourceGroup, machineName string) string {
|
||||
@ -700,7 +704,8 @@ func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node)
|
||||
|
||||
// GetPrimaryInterface gets machine primary network interface by node name.
|
||||
func (as *availabilitySet) GetPrimaryInterface(nodeName string) (network.Interface, error) {
|
||||
return as.getPrimaryInterfaceWithVMSet(nodeName, "")
|
||||
nic, _, err := as.getPrimaryInterfaceWithVMSet(nodeName, "")
|
||||
return nic, err
|
||||
}
|
||||
|
||||
// extractResourceGroupByNicID extracts the resource group name by nicID.
|
||||
@ -714,26 +719,26 @@ func extractResourceGroupByNicID(nicID string) (string, error) {
|
||||
}
|
||||
|
||||
// getPrimaryInterfaceWithVMSet gets machine primary network interface by node name and vmSet.
|
||||
func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, error) {
|
||||
func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, string, error) {
|
||||
var machine compute.VirtualMachine
|
||||
|
||||
machine, err := as.GetVirtualMachineWithRetry(types.NodeName(nodeName), azcache.CacheReadTypeDefault)
|
||||
if err != nil {
|
||||
klog.V(2).Infof("GetPrimaryInterface(%s, %s) abort backoff", nodeName, vmSetName)
|
||||
return network.Interface{}, err
|
||||
return network.Interface{}, "", err
|
||||
}
|
||||
|
||||
primaryNicID, err := getPrimaryInterfaceID(machine)
|
||||
if err != nil {
|
||||
return network.Interface{}, err
|
||||
return network.Interface{}, "", err
|
||||
}
|
||||
nicName, err := getLastSegment(primaryNicID, "/")
|
||||
if err != nil {
|
||||
return network.Interface{}, err
|
||||
return network.Interface{}, "", err
|
||||
}
|
||||
nodeResourceGroup, err := as.GetNodeResourceGroup(nodeName)
|
||||
if err != nil {
|
||||
return network.Interface{}, err
|
||||
return network.Interface{}, "", err
|
||||
}
|
||||
|
||||
// Check availability set name. Note that vmSetName is empty string when getting
|
||||
@ -748,23 +753,27 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
|
||||
if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) {
|
||||
klog.V(3).Infof(
|
||||
"GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName)
|
||||
return network.Interface{}, errNotInVMSet
|
||||
return network.Interface{}, "", errNotInVMSet
|
||||
}
|
||||
}
|
||||
|
||||
nicResourceGroup, err := extractResourceGroupByNicID(primaryNicID)
|
||||
if err != nil {
|
||||
return network.Interface{}, err
|
||||
return network.Interface{}, "", err
|
||||
}
|
||||
|
||||
ctx, cancel := getContextWithCancel()
|
||||
defer cancel()
|
||||
nic, rerr := as.InterfacesClient.Get(ctx, nicResourceGroup, nicName, "")
|
||||
if rerr != nil {
|
||||
return network.Interface{}, rerr.Error()
|
||||
return network.Interface{}, "", rerr.Error()
|
||||
}
|
||||
|
||||
return nic, nil
|
||||
var availabilitySetID string
|
||||
if machine.VirtualMachineProperties != nil && machine.AvailabilitySet != nil {
|
||||
availabilitySetID = to.String(machine.AvailabilitySet.ID)
|
||||
}
|
||||
return nic, availabilitySetID, nil
|
||||
}
|
||||
|
||||
// EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
|
||||
@ -772,7 +781,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
|
||||
func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
|
||||
vmName := mapNodeNameToVMName(nodeName)
|
||||
serviceName := getServiceName(service)
|
||||
nic, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
|
||||
nic, _, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
|
||||
if err != nil {
|
||||
if err == errNotInVMSet {
|
||||
klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName)
|
||||
@ -895,7 +904,111 @@ func (as *availabilitySet) EnsureHostsInPool(service *v1.Service, nodes []*v1.No
|
||||
|
||||
// EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes.
|
||||
func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error {
|
||||
// Do nothing for availability set.
|
||||
// Returns nil if backend address pools already deleted.
|
||||
if backendAddressPools == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
mc := metrics.NewMetricContext("services", "vmas_ensure_backend_pool_deleted", as.ResourceGroup, as.SubscriptionID, service.Name)
|
||||
isOperationSucceeded := false
|
||||
defer func() {
|
||||
mc.ObserveOperationWithResult(isOperationSucceeded)
|
||||
}()
|
||||
|
||||
ipConfigurationIDs := []string{}
|
||||
for _, backendPool := range *backendAddressPools {
|
||||
if strings.EqualFold(to.String(backendPool.ID), backendPoolID) &&
|
||||
backendPool.BackendAddressPoolPropertiesFormat != nil &&
|
||||
backendPool.BackendIPConfigurations != nil {
|
||||
for _, ipConf := range *backendPool.BackendIPConfigurations {
|
||||
if ipConf.ID == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
ipConfigurationIDs = append(ipConfigurationIDs, *ipConf.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
nicUpdaters := make([]func() error, 0)
|
||||
errors := make([]error, 0)
|
||||
for i := range ipConfigurationIDs {
|
||||
ipConfigurationID := ipConfigurationIDs[i]
|
||||
nodeName, err := as.GetNodeNameByIPConfigurationID(ipConfigurationID)
|
||||
if err != nil {
|
||||
klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
|
||||
errors = append(errors, err)
|
||||
continue
|
||||
}
|
||||
|
||||
vmName := mapNodeNameToVMName(types.NodeName(nodeName))
|
||||
nic, vmasID, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
|
||||
if err != nil {
|
||||
if err == errNotInVMSet {
|
||||
klog.V(3).Infof("EnsureBackendPoolDeleted skips node %s because it is not in the vmSet %s", nodeName, vmSetName)
|
||||
return nil
|
||||
}
|
||||
|
||||
klog.Errorf("error: az.EnsureBackendPoolDeleted(%s), az.VMSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err)
|
||||
return err
|
||||
}
|
||||
matches := vmasIDRE.FindStringSubmatch(vmasID)
|
||||
if len(matches) != 2 {
|
||||
return fmt.Errorf("EnsureBackendPoolDeleted: failed to parse the VMAS ID %s: %v", vmasID, err)
|
||||
}
|
||||
vmasName := matches[1]
|
||||
// Only remove nodes belonging to specified vmSet to basic LB backends.
|
||||
if !strings.EqualFold(vmasName, vmSetName) {
|
||||
klog.V(2).Infof("EnsureBackendPoolDeleted: skipping the node %s belonging to another vm set %s", nodeName, vmasName)
|
||||
continue
|
||||
}
|
||||
|
||||
if nic.ProvisioningState != nil && *nic.ProvisioningState == nicFailedState {
|
||||
klog.Warningf("EnsureBackendPoolDeleted skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
if nic.InterfacePropertiesFormat != nil && nic.InterfacePropertiesFormat.IPConfigurations != nil {
|
||||
newIPConfigs := *nic.IPConfigurations
|
||||
for j, ipConf := range newIPConfigs {
|
||||
if !to.Bool(ipConf.Primary) {
|
||||
continue
|
||||
}
|
||||
// found primary ip configuration
|
||||
if ipConf.LoadBalancerBackendAddressPools != nil {
|
||||
newLBAddressPools := *ipConf.LoadBalancerBackendAddressPools
|
||||
for k, pool := range newLBAddressPools {
|
||||
if strings.EqualFold(to.String(pool.ID), backendPoolID) {
|
||||
newLBAddressPools = append(newLBAddressPools[:k], newLBAddressPools[k+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
newIPConfigs[j].LoadBalancerBackendAddressPools = &newLBAddressPools
|
||||
}
|
||||
}
|
||||
nic.IPConfigurations = &newIPConfigs
|
||||
nicUpdaters = append(nicUpdaters, func() error {
|
||||
ctx, cancel := getContextWithCancel()
|
||||
defer cancel()
|
||||
klog.V(2).Infof("EnsureBackendPoolDeleted begins to CreateOrUpdate for NIC(%s, %s) with backendPoolID %s", as.resourceGroup, to.String(nic.Name), backendPoolID)
|
||||
rerr := as.InterfacesClient.CreateOrUpdate(ctx, as.ResourceGroup, to.String(nic.Name), nic)
|
||||
if rerr != nil {
|
||||
klog.Errorf("EnsureBackendPoolDeleted CreateOrUpdate for NIC(%s, %s) failed with error %v", as.resourceGroup, to.String(nic.Name), rerr.Error())
|
||||
return rerr.Error()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
errs := utilerrors.AggregateGoroutines(nicUpdaters...)
|
||||
if errs != nil {
|
||||
return utilerrors.Flatten(errs)
|
||||
}
|
||||
// Fail if there are other errors.
|
||||
if len(errors) > 0 {
|
||||
return utilerrors.Flatten(utilerrors.NewAggregate(errors))
|
||||
}
|
||||
|
||||
isOperationSucceeded = true
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -908,3 +1021,16 @@ func generateStorageAccountName(accountNamePrefix string) string {
|
||||
}
|
||||
return accountName
|
||||
}
|
||||
|
||||
// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
|
||||
func (as *availabilitySet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
|
||||
matches := nicIDRE.FindStringSubmatch(ipConfigurationID)
|
||||
if len(matches) != 3 {
|
||||
klog.V(4).Infof("Can not extract VM name from ipConfigurationID (%s)", ipConfigurationID)
|
||||
return "", fmt.Errorf("invalid ip config ID %s", ipConfigurationID)
|
||||
}
|
||||
|
||||
prefix := matches[1]
|
||||
suffix := matches[2]
|
||||
return fmt.Sprintf("%s-%s", prefix, suffix), nil
|
||||
}
|
||||
|
@ -1665,3 +1665,81 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
|
||||
assert.Equal(t, test.isPrimary, isPrimary, test.desc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandardEnsureBackendPoolDeleted(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
cloud := GetTestCloud(ctrl)
|
||||
service := getTestService("test", v1.ProtocolTCP, nil, false, 80)
|
||||
backendPoolID := "backendPoolID"
|
||||
vmSetName := "AS"
|
||||
|
||||
tests := []struct {
|
||||
desc string
|
||||
backendAddressPools *[]network.BackendAddressPool
|
||||
loadBalancerSKU string
|
||||
existingVM compute.VirtualMachine
|
||||
existingNIC network.Interface
|
||||
}{
|
||||
{
|
||||
desc: "",
|
||||
backendAddressPools: &[]network.BackendAddressPool{
|
||||
{
|
||||
ID: to.StringPtr(backendPoolID),
|
||||
BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
|
||||
BackendIPConfigurations: &[]network.InterfaceIPConfiguration{
|
||||
{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
existingVM: compute.VirtualMachine{
|
||||
VirtualMachineProperties: &compute.VirtualMachineProperties{
|
||||
AvailabilitySet: &compute.SubResource{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/as"),
|
||||
},
|
||||
NetworkProfile: &compute.NetworkProfile{
|
||||
NetworkInterfaces: &[]compute.NetworkInterfaceReference{
|
||||
{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
existingNIC: network.Interface{
|
||||
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
|
||||
ProvisioningState: to.StringPtr("Succeeded"),
|
||||
IPConfigurations: &[]network.InterfaceIPConfiguration{
|
||||
{
|
||||
InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
|
||||
Primary: to.BoolPtr(true),
|
||||
LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
|
||||
{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
cloud.LoadBalancerSku = test.loadBalancerSKU
|
||||
mockVMClient := mockvmclient.NewMockInterface(ctrl)
|
||||
mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "k8s-agentpool1-00000000-1", gomock.Any()).Return(test.existingVM, nil)
|
||||
cloud.VirtualMachinesClient = mockVMClient
|
||||
mockNICClient := mockinterfaceclient.NewMockInterface(ctrl)
|
||||
mockNICClient.EXPECT().Get(gomock.Any(), "rg", "k8s-agentpool1-00000000-nic-1", gomock.Any()).Return(test.existingNIC, nil)
|
||||
mockNICClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
|
||||
cloud.InterfacesClient = mockNICClient
|
||||
|
||||
err := cloud.VMSet.EnsureBackendPoolDeleted(&service, backendPoolID, vmSetName, test.backendAddressPools)
|
||||
assert.NoError(t, err, test.desc)
|
||||
}
|
||||
}
|
||||
|
@ -78,4 +78,7 @@ type VMSet interface {
|
||||
|
||||
// GetPrivateIPsByNodeName returns a slice of all private ips assigned to node (ipv6 and ipv4)
|
||||
GetPrivateIPsByNodeName(name string) ([]string, error)
|
||||
|
||||
// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
|
||||
GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error)
|
||||
}
|
||||
|
@ -1297,7 +1297,7 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac
|
||||
|
||||
// ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted
|
||||
// from the specified node, which returns (resourceGroup, vmssName, instanceID, vmssVM, error).
|
||||
func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
|
||||
func (ss *scaleSet) ensureBackendPoolDeletedFromNode(nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
|
||||
ssName, instanceID, vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault)
|
||||
if err != nil {
|
||||
return "", "", "", nil, err
|
||||
@ -1363,8 +1363,8 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeNa
|
||||
return nodeResourceGroup, ssName, instanceID, newVM, nil
|
||||
}
|
||||
|
||||
// getNodeNameByIPConfigurationID gets the node name by IP configuration ID.
|
||||
func (ss *scaleSet) getNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
|
||||
// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
|
||||
func (ss *scaleSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
|
||||
matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
|
||||
if len(matches) != 4 {
|
||||
klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is mananaged by availability set", ipConfigurationID)
|
||||
@ -1529,18 +1529,18 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID,
|
||||
}
|
||||
}
|
||||
|
||||
nodeName, err := ss.getNodeNameByIPConfigurationID(ipConfigurationID)
|
||||
nodeName, err := ss.GetNodeNameByIPConfigurationID(ipConfigurationID)
|
||||
if err != nil {
|
||||
if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes.
|
||||
continue
|
||||
}
|
||||
|
||||
klog.Errorf("Failed to getNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
|
||||
klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
|
||||
errors = append(errors, err)
|
||||
continue
|
||||
}
|
||||
|
||||
nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(service, nodeName, backendPoolID)
|
||||
nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(nodeName, backendPoolID)
|
||||
if err != nil {
|
||||
klog.Errorf("EnsureBackendPoolDeleted(%s): backendPoolID(%s) - failed with error %v", getServiceName(service), backendPoolID, err)
|
||||
errors = append(errors, err)
|
||||
|
@ -581,21 +581,21 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
description: "getNodeNameByIPConfigurationID should get node's Name when the node is existing",
|
||||
description: "GetNodeNameByIPConfigurationID should get node's Name when the node is existing",
|
||||
scaleSet: "scaleset1",
|
||||
ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset1", "0", "scaleset1"),
|
||||
vmList: []string{"vmssee6c2000000", "vmssee6c2000001"},
|
||||
expected: "vmssee6c2000000",
|
||||
},
|
||||
{
|
||||
description: "getNodeNameByIPConfigurationID should return error for non-exist nodes",
|
||||
description: "GetNodeNameByIPConfigurationID should return error for non-exist nodes",
|
||||
scaleSet: "scaleset2",
|
||||
ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset2", "3", "scaleset1"),
|
||||
vmList: []string{"vmssee6c2000002", "vmssee6c2000003"},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
description: "getNodeNameByIPConfigurationID should return error for wrong ipConfigurationID",
|
||||
description: "GetNodeNameByIPConfigurationID should return error for wrong ipConfigurationID",
|
||||
scaleSet: "scaleset3",
|
||||
ipConfigurationID: "invalid-configuration-id",
|
||||
vmList: []string{"vmssee6c2000004", "vmssee6c2000005"},
|
||||
@ -618,7 +618,7 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
|
||||
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
|
||||
mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes()
|
||||
|
||||
nodeName, err := ss.getNodeNameByIPConfigurationID(test.ipConfigurationID)
|
||||
nodeName, err := ss.GetNodeNameByIPConfigurationID(test.ipConfigurationID)
|
||||
if test.expectError {
|
||||
assert.Error(t, err, test.description)
|
||||
continue
|
||||
@ -2113,7 +2113,7 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) {
|
||||
mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
|
||||
mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
|
||||
|
||||
nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(&v1.Service{}, test.nodeName, test.backendpoolID)
|
||||
nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(test.nodeName, test.backendpoolID)
|
||||
assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs")
|
||||
assert.Equal(t, test.expectedNodeResourceGroup, nodeResourceGroup, test.description)
|
||||
assert.Equal(t, test.expectedVMSSName, ssName, test.description)
|
||||
|
@ -291,3 +291,18 @@ func (mr *MockVMSetMockRecorder) GetPrivateIPsByNodeName(name interface{}) *gomo
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrivateIPsByNodeName", reflect.TypeOf((*MockVMSet)(nil).GetPrivateIPsByNodeName), name)
|
||||
}
|
||||
|
||||
// GetNodeNameByIPConfigurationID mocks base method
|
||||
func (m *MockVMSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetNodeNameByIPConfigurationID", ipConfigurationID)
|
||||
ret0, _ := ret[0].(string)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetNodeNameByIPConfigurationID indicates an expected call of GetNodeNameByIPConfigurationID
|
||||
func (mr *MockVMSetMockRecorder) GetNodeNameByIPConfigurationID(ipConfigurationID interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeNameByIPConfigurationID", reflect.TypeOf((*MockVMSet)(nil).GetNodeNameByIPConfigurationID), ipConfigurationID)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user