Merge pull request #95542 from nilo19/bug/support-exclude-node-from-lb-label

Support the node label `node.kubernetes.io/exclude-from-external-load-balancers`
This commit is contained in:
Kubernetes Prow Robot 2020-10-15 01:38:24 -07:00 committed by GitHub
commit d7e0cb0e35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 308 additions and 40 deletions

View File

@ -981,6 +981,15 @@ func (az *Cloud) findFrontendIPConfigOfService(
return nil, false, nil return nil, false, nil
} }
func nodeNameInNodes(nodeName string, nodes []*v1.Node) bool {
for _, node := range nodes {
if strings.EqualFold(nodeName, node.Name) {
return true
}
}
return false
}
// This ensures load balancer exists and the frontend ip config is setup. // This ensures load balancer exists and the frontend ip config is setup.
// This also reconciles the Service's Ports with the LoadBalancer config. // This also reconciles the Service's Ports with the LoadBalancer config.
// This entails adding rules/probes for expected Ports and removing stale rules/ports. // This entails adding rules/probes for expected Ports and removing stale rules/ports.
@ -1022,6 +1031,42 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
if strings.EqualFold(*bp.Name, lbBackendPoolName) { if strings.EqualFold(*bp.Name, lbBackendPoolName) {
klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb) klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb)
foundBackendPool = true foundBackendPool = true
var backendIPConfigurationsToBeDeleted []network.InterfaceIPConfiguration
if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil {
for _, ipConf := range *bp.BackendIPConfigurations {
ipConfID := to.String(ipConf.ID)
nodeName, err := az.VMSet.GetNodeNameByIPConfigurationID(ipConfID)
if err != nil {
return nil, err
}
// If a node is not supposed to be included in the LB, it
// would not be in the `nodes` slice. We need to check the nodes that
// have been added to the LB's backendpool, find the unwanted ones and
// delete them from the pool.
if !nodeNameInNodes(nodeName, nodes) {
klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found unwanted node %s, decouple it from the LB", serviceName, wantLb, nodeName)
// construct a backendPool that only contains the IP config of the node to be deleted
backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, network.InterfaceIPConfiguration{ID: to.StringPtr(ipConfID)})
}
}
if len(backendIPConfigurationsToBeDeleted) > 0 {
backendpoolToBeDeleted := &[]network.BackendAddressPool{
{
ID: to.StringPtr(lbBackendPoolID),
BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
BackendIPConfigurations: &backendIPConfigurationsToBeDeleted,
},
},
}
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
// decouple the backendPool from the node
err = az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, backendpoolToBeDeleted)
if err != nil {
return nil, err
}
}
}
break break
} else { } else {
klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found other backendpool %s", serviceName, wantLb, *bp.Name) klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found other backendpool %s", serviceName, wantLb, *bp.Name)
@ -1294,6 +1339,10 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB. // Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName) klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
if _, ok := az.VMSet.(*availabilitySet); ok {
// do nothing for availability set
lb.BackendAddressPools = nil
}
err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools) err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
if err != nil { if err != nil {
klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err) klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
@ -2170,8 +2219,8 @@ func equalLoadBalancingRulePropertiesFormat(s *network.LoadBalancingRuleProperti
reflect.DeepEqual(s.FrontendPort, t.FrontendPort) && reflect.DeepEqual(s.FrontendPort, t.FrontendPort) &&
reflect.DeepEqual(s.BackendPort, t.BackendPort) && reflect.DeepEqual(s.BackendPort, t.BackendPort) &&
reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP) && reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP) &&
reflect.DeepEqual(s.EnableTCPReset, t.EnableTCPReset) && reflect.DeepEqual(to.Bool(s.EnableTCPReset), to.Bool(t.EnableTCPReset)) &&
reflect.DeepEqual(s.DisableOutboundSnat, t.DisableOutboundSnat) reflect.DeepEqual(to.Bool(s.DisableOutboundSnat), to.Bool(t.DisableOutboundSnat))
if wantLB && s.IdleTimeoutInMinutes != nil && t.IdleTimeoutInMinutes != nil { if wantLB && s.IdleTimeoutInMinutes != nil && t.IdleTimeoutInMinutes != nil {
return properties && reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes) return properties && reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes)

View File

@ -1762,11 +1762,12 @@ func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" +
"Microsoft.Network/loadBalancers/" + *name + "/backendAddressPools/" + *clusterName), "Microsoft.Network/loadBalancers/" + *name + "/backendAddressPools/" + *clusterName),
}, },
LoadDistribution: network.LoadDistribution("Default"), LoadDistribution: network.LoadDistribution("Default"),
FrontendPort: to.Int32Ptr(service.Spec.Ports[0].Port), FrontendPort: to.Int32Ptr(service.Spec.Ports[0].Port),
BackendPort: to.Int32Ptr(service.Spec.Ports[0].Port), BackendPort: to.Int32Ptr(service.Spec.Ports[0].Port),
EnableFloatingIP: to.BoolPtr(true), EnableFloatingIP: to.BoolPtr(true),
EnableTCPReset: to.BoolPtr(strings.EqualFold(lbSku, "standard")), EnableTCPReset: to.BoolPtr(strings.EqualFold(lbSku, "standard")),
DisableOutboundSnat: to.BoolPtr(false),
Probe: &network.SubResource{ Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80"), ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80"),
}, },
@ -1833,8 +1834,6 @@ func TestReconcileLoadBalancer(t *testing.T) {
}, },
} }
expectedLb1 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service3, "Basic") expectedLb1 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service3, "Basic")
(*expectedLb1.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false)
(*expectedLb1.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
expectedLb1.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{ expectedLb1.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
{ {
Name: to.StringPtr("aservice1"), Name: to.StringPtr("aservice1"),
@ -1970,9 +1969,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
lb6.Probes = &[]network.Probe{} lb6.Probes = &[]network.Probe{}
expectedLB6 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service6, "basic") expectedLB6 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service6, "basic")
expectedLB6.Probes = &[]network.Probe{} expectedLB6.Probes = &[]network.Probe{}
(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].Probe = nil (*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].Probe = &network.SubResource{ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80")}
(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false)
expectedLB6.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{ expectedLB6.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
{ {
Name: to.StringPtr("aservice1"), Name: to.StringPtr("aservice1"),
@ -1994,7 +1991,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-UDP-80"), ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-UDP-80"),
} }
(*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil (*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
(*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false) (*lb7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(true)
expectedLB7.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{ expectedLB7.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
{ {
Name: to.StringPtr("aservice1"), Name: to.StringPtr("aservice1"),

View File

@ -71,10 +71,14 @@ const (
loadBalancerRuleNameMaxLength = 80 loadBalancerRuleNameMaxLength = 80
) )
var errNotInVMSet = errors.New("vm is not in the vmset") var (
var providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`) errNotInVMSet = errors.New("vm is not in the vmset")
var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`) providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
var nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`) backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`)
nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`)
nicIDRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.+)/providers/Microsoft.Network/networkInterfaces/(.+)-nic-(.+)/ipConfigurations/(?:.*)`)
vmasIDRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)`)
)
// getStandardMachineID returns the full identifier of a virtual machine. // getStandardMachineID returns the full identifier of a virtual machine.
func (az *Cloud) getStandardMachineID(subscriptionID, resourceGroup, machineName string) string { func (az *Cloud) getStandardMachineID(subscriptionID, resourceGroup, machineName string) string {
@ -700,7 +704,8 @@ func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node)
// GetPrimaryInterface gets machine primary network interface by node name. // GetPrimaryInterface gets machine primary network interface by node name.
func (as *availabilitySet) GetPrimaryInterface(nodeName string) (network.Interface, error) { func (as *availabilitySet) GetPrimaryInterface(nodeName string) (network.Interface, error) {
return as.getPrimaryInterfaceWithVMSet(nodeName, "") nic, _, err := as.getPrimaryInterfaceWithVMSet(nodeName, "")
return nic, err
} }
// extractResourceGroupByNicID extracts the resource group name by nicID. // extractResourceGroupByNicID extracts the resource group name by nicID.
@ -714,26 +719,26 @@ func extractResourceGroupByNicID(nicID string) (string, error) {
} }
// getPrimaryInterfaceWithVMSet gets machine primary network interface by node name and vmSet. // getPrimaryInterfaceWithVMSet gets machine primary network interface by node name and vmSet.
func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, error) { func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, string, error) {
var machine compute.VirtualMachine var machine compute.VirtualMachine
machine, err := as.GetVirtualMachineWithRetry(types.NodeName(nodeName), azcache.CacheReadTypeDefault) machine, err := as.GetVirtualMachineWithRetry(types.NodeName(nodeName), azcache.CacheReadTypeDefault)
if err != nil { if err != nil {
klog.V(2).Infof("GetPrimaryInterface(%s, %s) abort backoff", nodeName, vmSetName) klog.V(2).Infof("GetPrimaryInterface(%s, %s) abort backoff", nodeName, vmSetName)
return network.Interface{}, err return network.Interface{}, "", err
} }
primaryNicID, err := getPrimaryInterfaceID(machine) primaryNicID, err := getPrimaryInterfaceID(machine)
if err != nil { if err != nil {
return network.Interface{}, err return network.Interface{}, "", err
} }
nicName, err := getLastSegment(primaryNicID, "/") nicName, err := getLastSegment(primaryNicID, "/")
if err != nil { if err != nil {
return network.Interface{}, err return network.Interface{}, "", err
} }
nodeResourceGroup, err := as.GetNodeResourceGroup(nodeName) nodeResourceGroup, err := as.GetNodeResourceGroup(nodeName)
if err != nil { if err != nil {
return network.Interface{}, err return network.Interface{}, "", err
} }
// Check availability set name. Note that vmSetName is empty string when getting // Check availability set name. Note that vmSetName is empty string when getting
@ -748,23 +753,27 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) { if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) {
klog.V(3).Infof( klog.V(3).Infof(
"GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName) "GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName)
return network.Interface{}, errNotInVMSet return network.Interface{}, "", errNotInVMSet
} }
} }
nicResourceGroup, err := extractResourceGroupByNicID(primaryNicID) nicResourceGroup, err := extractResourceGroupByNicID(primaryNicID)
if err != nil { if err != nil {
return network.Interface{}, err return network.Interface{}, "", err
} }
ctx, cancel := getContextWithCancel() ctx, cancel := getContextWithCancel()
defer cancel() defer cancel()
nic, rerr := as.InterfacesClient.Get(ctx, nicResourceGroup, nicName, "") nic, rerr := as.InterfacesClient.Get(ctx, nicResourceGroup, nicName, "")
if rerr != nil { if rerr != nil {
return network.Interface{}, rerr.Error() return network.Interface{}, "", rerr.Error()
} }
return nic, nil var availabilitySetID string
if machine.VirtualMachineProperties != nil && machine.AvailabilitySet != nil {
availabilitySetID = to.String(machine.AvailabilitySet.ID)
}
return nic, availabilitySetID, nil
} }
// EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
@ -772,7 +781,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
vmName := mapNodeNameToVMName(nodeName) vmName := mapNodeNameToVMName(nodeName)
serviceName := getServiceName(service) serviceName := getServiceName(service)
nic, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName) nic, _, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
if err != nil { if err != nil {
if err == errNotInVMSet { if err == errNotInVMSet {
klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName) klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName)
@ -895,7 +904,111 @@ func (as *availabilitySet) EnsureHostsInPool(service *v1.Service, nodes []*v1.No
// EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes. // EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes.
func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error { func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error {
// Do nothing for availability set. // Returns nil if backend address pools already deleted.
if backendAddressPools == nil {
return nil
}
mc := metrics.NewMetricContext("services", "vmas_ensure_backend_pool_deleted", as.ResourceGroup, as.SubscriptionID, service.Name)
isOperationSucceeded := false
defer func() {
mc.ObserveOperationWithResult(isOperationSucceeded)
}()
ipConfigurationIDs := []string{}
for _, backendPool := range *backendAddressPools {
if strings.EqualFold(to.String(backendPool.ID), backendPoolID) &&
backendPool.BackendAddressPoolPropertiesFormat != nil &&
backendPool.BackendIPConfigurations != nil {
for _, ipConf := range *backendPool.BackendIPConfigurations {
if ipConf.ID == nil {
continue
}
ipConfigurationIDs = append(ipConfigurationIDs, *ipConf.ID)
}
}
}
nicUpdaters := make([]func() error, 0)
errors := make([]error, 0)
for i := range ipConfigurationIDs {
ipConfigurationID := ipConfigurationIDs[i]
nodeName, err := as.GetNodeNameByIPConfigurationID(ipConfigurationID)
if err != nil {
klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
errors = append(errors, err)
continue
}
vmName := mapNodeNameToVMName(types.NodeName(nodeName))
nic, vmasID, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
if err != nil {
if err == errNotInVMSet {
klog.V(3).Infof("EnsureBackendPoolDeleted skips node %s because it is not in the vmSet %s", nodeName, vmSetName)
return nil
}
klog.Errorf("error: az.EnsureBackendPoolDeleted(%s), az.VMSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err)
return err
}
matches := vmasIDRE.FindStringSubmatch(vmasID)
if len(matches) != 2 {
return fmt.Errorf("EnsureBackendPoolDeleted: failed to parse the VMAS ID %s: %v", vmasID, err)
}
vmasName := matches[1]
// Only remove nodes belonging to specified vmSet to basic LB backends.
if !strings.EqualFold(vmasName, vmSetName) {
klog.V(2).Infof("EnsureBackendPoolDeleted: skipping the node %s belonging to another vm set %s", nodeName, vmasName)
continue
}
if nic.ProvisioningState != nil && *nic.ProvisioningState == nicFailedState {
klog.Warningf("EnsureBackendPoolDeleted skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name)
return nil
}
if nic.InterfacePropertiesFormat != nil && nic.InterfacePropertiesFormat.IPConfigurations != nil {
newIPConfigs := *nic.IPConfigurations
for j, ipConf := range newIPConfigs {
if !to.Bool(ipConf.Primary) {
continue
}
// found primary ip configuration
if ipConf.LoadBalancerBackendAddressPools != nil {
newLBAddressPools := *ipConf.LoadBalancerBackendAddressPools
for k, pool := range newLBAddressPools {
if strings.EqualFold(to.String(pool.ID), backendPoolID) {
newLBAddressPools = append(newLBAddressPools[:k], newLBAddressPools[k+1:]...)
break
}
}
newIPConfigs[j].LoadBalancerBackendAddressPools = &newLBAddressPools
}
}
nic.IPConfigurations = &newIPConfigs
nicUpdaters = append(nicUpdaters, func() error {
ctx, cancel := getContextWithCancel()
defer cancel()
klog.V(2).Infof("EnsureBackendPoolDeleted begins to CreateOrUpdate for NIC(%s, %s) with backendPoolID %s", as.resourceGroup, to.String(nic.Name), backendPoolID)
rerr := as.InterfacesClient.CreateOrUpdate(ctx, as.ResourceGroup, to.String(nic.Name), nic)
if rerr != nil {
klog.Errorf("EnsureBackendPoolDeleted CreateOrUpdate for NIC(%s, %s) failed with error %v", as.resourceGroup, to.String(nic.Name), rerr.Error())
return rerr.Error()
}
return nil
})
}
}
errs := utilerrors.AggregateGoroutines(nicUpdaters...)
if errs != nil {
return utilerrors.Flatten(errs)
}
// Fail if there are other errors.
if len(errors) > 0 {
return utilerrors.Flatten(utilerrors.NewAggregate(errors))
}
isOperationSucceeded = true
return nil return nil
} }
@ -908,3 +1021,16 @@ func generateStorageAccountName(accountNamePrefix string) string {
} }
return accountName return accountName
} }
// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
func (as *availabilitySet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
matches := nicIDRE.FindStringSubmatch(ipConfigurationID)
if len(matches) != 3 {
klog.V(4).Infof("Can not extract VM name from ipConfigurationID (%s)", ipConfigurationID)
return "", fmt.Errorf("invalid ip config ID %s", ipConfigurationID)
}
prefix := matches[1]
suffix := matches[2]
return fmt.Sprintf("%s-%s", prefix, suffix), nil
}

View File

@ -1665,3 +1665,81 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
assert.Equal(t, test.isPrimary, isPrimary, test.desc) assert.Equal(t, test.isPrimary, isPrimary, test.desc)
} }
} }
func TestStandardEnsureBackendPoolDeleted(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cloud := GetTestCloud(ctrl)
service := getTestService("test", v1.ProtocolTCP, nil, false, 80)
backendPoolID := "backendPoolID"
vmSetName := "AS"
tests := []struct {
desc string
backendAddressPools *[]network.BackendAddressPool
loadBalancerSKU string
existingVM compute.VirtualMachine
existingNIC network.Interface
}{
{
desc: "",
backendAddressPools: &[]network.BackendAddressPool{
{
ID: to.StringPtr(backendPoolID),
BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
BackendIPConfigurations: &[]network.InterfaceIPConfiguration{
{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"),
},
},
},
},
},
existingVM: compute.VirtualMachine{
VirtualMachineProperties: &compute.VirtualMachineProperties{
AvailabilitySet: &compute.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/as"),
},
NetworkProfile: &compute.NetworkProfile{
NetworkInterfaces: &[]compute.NetworkInterfaceReference{
{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1"),
},
},
},
},
},
existingNIC: network.Interface{
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
ProvisioningState: to.StringPtr("Succeeded"),
IPConfigurations: &[]network.InterfaceIPConfiguration{
{
InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
Primary: to.BoolPtr(true),
LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"),
},
},
},
},
},
},
},
},
}
for _, test := range tests {
cloud.LoadBalancerSku = test.loadBalancerSKU
mockVMClient := mockvmclient.NewMockInterface(ctrl)
mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "k8s-agentpool1-00000000-1", gomock.Any()).Return(test.existingVM, nil)
cloud.VirtualMachinesClient = mockVMClient
mockNICClient := mockinterfaceclient.NewMockInterface(ctrl)
mockNICClient.EXPECT().Get(gomock.Any(), "rg", "k8s-agentpool1-00000000-nic-1", gomock.Any()).Return(test.existingNIC, nil)
mockNICClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
cloud.InterfacesClient = mockNICClient
err := cloud.VMSet.EnsureBackendPoolDeleted(&service, backendPoolID, vmSetName, test.backendAddressPools)
assert.NoError(t, err, test.desc)
}
}

View File

@ -78,4 +78,7 @@ type VMSet interface {
// GetPrivateIPsByNodeName returns a slice of all private ips assigned to node (ipv6 and ipv4) // GetPrivateIPsByNodeName returns a slice of all private ips assigned to node (ipv6 and ipv4)
GetPrivateIPsByNodeName(name string) ([]string, error) GetPrivateIPsByNodeName(name string) ([]string, error)
// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error)
} }

View File

@ -1297,7 +1297,7 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac
// ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted // ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted
// from the specified node, which returns (resourceGroup, vmssName, instanceID, vmssVM, error). // from the specified node, which returns (resourceGroup, vmssName, instanceID, vmssVM, error).
func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { func (ss *scaleSet) ensureBackendPoolDeletedFromNode(nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
ssName, instanceID, vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault) ssName, instanceID, vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault)
if err != nil { if err != nil {
return "", "", "", nil, err return "", "", "", nil, err
@ -1363,8 +1363,8 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeNa
return nodeResourceGroup, ssName, instanceID, newVM, nil return nodeResourceGroup, ssName, instanceID, newVM, nil
} }
// getNodeNameByIPConfigurationID gets the node name by IP configuration ID. // GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
func (ss *scaleSet) getNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) { func (ss *scaleSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID) matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
if len(matches) != 4 { if len(matches) != 4 {
klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is mananaged by availability set", ipConfigurationID) klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is mananaged by availability set", ipConfigurationID)
@ -1529,18 +1529,18 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID,
} }
} }
nodeName, err := ss.getNodeNameByIPConfigurationID(ipConfigurationID) nodeName, err := ss.GetNodeNameByIPConfigurationID(ipConfigurationID)
if err != nil { if err != nil {
if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes. if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes.
continue continue
} }
klog.Errorf("Failed to getNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err) klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
errors = append(errors, err) errors = append(errors, err)
continue continue
} }
nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(service, nodeName, backendPoolID) nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(nodeName, backendPoolID)
if err != nil { if err != nil {
klog.Errorf("EnsureBackendPoolDeleted(%s): backendPoolID(%s) - failed with error %v", getServiceName(service), backendPoolID, err) klog.Errorf("EnsureBackendPoolDeleted(%s): backendPoolID(%s) - failed with error %v", getServiceName(service), backendPoolID, err)
errors = append(errors, err) errors = append(errors, err)

View File

@ -581,21 +581,21 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
expectError bool expectError bool
}{ }{
{ {
description: "getNodeNameByIPConfigurationID should get node's Name when the node is existing", description: "GetNodeNameByIPConfigurationID should get node's Name when the node is existing",
scaleSet: "scaleset1", scaleSet: "scaleset1",
ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset1", "0", "scaleset1"), ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset1", "0", "scaleset1"),
vmList: []string{"vmssee6c2000000", "vmssee6c2000001"}, vmList: []string{"vmssee6c2000000", "vmssee6c2000001"},
expected: "vmssee6c2000000", expected: "vmssee6c2000000",
}, },
{ {
description: "getNodeNameByIPConfigurationID should return error for non-exist nodes", description: "GetNodeNameByIPConfigurationID should return error for non-exist nodes",
scaleSet: "scaleset2", scaleSet: "scaleset2",
ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset2", "3", "scaleset1"), ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset2", "3", "scaleset1"),
vmList: []string{"vmssee6c2000002", "vmssee6c2000003"}, vmList: []string{"vmssee6c2000002", "vmssee6c2000003"},
expectError: true, expectError: true,
}, },
{ {
description: "getNodeNameByIPConfigurationID should return error for wrong ipConfigurationID", description: "GetNodeNameByIPConfigurationID should return error for wrong ipConfigurationID",
scaleSet: "scaleset3", scaleSet: "scaleset3",
ipConfigurationID: "invalid-configuration-id", ipConfigurationID: "invalid-configuration-id",
vmList: []string{"vmssee6c2000004", "vmssee6c2000005"}, vmList: []string{"vmssee6c2000004", "vmssee6c2000005"},
@ -618,7 +618,7 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes()
nodeName, err := ss.getNodeNameByIPConfigurationID(test.ipConfigurationID) nodeName, err := ss.GetNodeNameByIPConfigurationID(test.ipConfigurationID)
if test.expectError { if test.expectError {
assert.Error(t, err, test.description) assert.Error(t, err, test.description)
continue continue
@ -2113,7 +2113,7 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) {
mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(&v1.Service{}, test.nodeName, test.backendpoolID) nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(test.nodeName, test.backendpoolID)
assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs")
assert.Equal(t, test.expectedNodeResourceGroup, nodeResourceGroup, test.description) assert.Equal(t, test.expectedNodeResourceGroup, nodeResourceGroup, test.description)
assert.Equal(t, test.expectedVMSSName, ssName, test.description) assert.Equal(t, test.expectedVMSSName, ssName, test.description)

View File

@ -291,3 +291,18 @@ func (mr *MockVMSetMockRecorder) GetPrivateIPsByNodeName(name interface{}) *gomo
mr.mock.ctrl.T.Helper() mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrivateIPsByNodeName", reflect.TypeOf((*MockVMSet)(nil).GetPrivateIPsByNodeName), name) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrivateIPsByNodeName", reflect.TypeOf((*MockVMSet)(nil).GetPrivateIPsByNodeName), name)
} }
// GetNodeNameByIPConfigurationID mocks base method
func (m *MockVMSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetNodeNameByIPConfigurationID", ipConfigurationID)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetNodeNameByIPConfigurationID indicates an expected call of GetNodeNameByIPConfigurationID
func (mr *MockVMSetMockRecorder) GetNodeNameByIPConfigurationID(ipConfigurationID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeNameByIPConfigurationID", reflect.TypeOf((*MockVMSet)(nil).GetNodeNameByIPConfigurationID), ipConfigurationID)
}