mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-26 21:17:23 +00:00
Merge pull request #96111 from nilo19/feature/support-multiple-slb
Azure: Support multiple standard load balancers in one cluster
This commit is contained in:
commit
3e7e6c3455
@ -197,6 +197,12 @@ type Config struct {
|
|||||||
// "external": for external LoadBalancer
|
// "external": for external LoadBalancer
|
||||||
// "all": for both internal and external LoadBalancer
|
// "all": for both internal and external LoadBalancer
|
||||||
PreConfiguredBackendPoolLoadBalancerTypes string `json:"preConfiguredBackendPoolLoadBalancerTypes,omitempty" yaml:"preConfiguredBackendPoolLoadBalancerTypes,omitempty"`
|
PreConfiguredBackendPoolLoadBalancerTypes string `json:"preConfiguredBackendPoolLoadBalancerTypes,omitempty" yaml:"preConfiguredBackendPoolLoadBalancerTypes,omitempty"`
|
||||||
|
// EnableMultipleStandardLoadBalancers determines the behavior of the standard load balancer. If set to true
|
||||||
|
// there would be one standard load balancer per VMAS or VMSS, which is similar with the behavior of the basic
|
||||||
|
// load balancer. Users could select the specific standard load balancer for their service by the service
|
||||||
|
// annotation `service.beta.kubernetes.io/azure-load-balancer-mode`, If set to false, the same standard load balancer
|
||||||
|
// would be shared by all services in the cluster. In this case, the mode selection annotation would be ignored.
|
||||||
|
EnableMultipleStandardLoadBalancers bool `json:"enableMultipleStandardLoadBalancers,omitempty" yaml:"enableMultipleStandardLoadBalancers,omitempty"`
|
||||||
|
|
||||||
// AvailabilitySetNodesCacheTTLInSeconds sets the Cache TTL for availabilitySetNodesCache
|
// AvailabilitySetNodesCacheTTLInSeconds sets the Cache TTL for availabilitySetNodesCache
|
||||||
// if not set, will use default value
|
// if not set, will use default value
|
||||||
|
@ -274,6 +274,65 @@ func (az *Cloud) getLoadBalancerResourceGroup() string {
|
|||||||
return az.ResourceGroup
|
return az.ResourceGroup
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cleanBackendpoolForPrimarySLB decouples the unwanted nodes from the standard load balancer.
|
||||||
|
// This is needed because when migrating from single SLB to multiple SLBs, The existing
|
||||||
|
// SLB's backend pool contains nodes from different agent pools, while we only want the
|
||||||
|
// nodes from the primary agent pool to join the backend pool.
|
||||||
|
func (az *Cloud) cleanBackendpoolForPrimarySLB(primarySLB *network.LoadBalancer, service *v1.Service, clusterName string) (*network.LoadBalancer, error) {
|
||||||
|
lbBackendPoolName := getBackendPoolName(clusterName, service)
|
||||||
|
lbResourceGroup := az.getLoadBalancerResourceGroup()
|
||||||
|
lbBackendPoolID := az.getBackendPoolID(to.String(primarySLB.Name), lbResourceGroup, lbBackendPoolName)
|
||||||
|
newBackendPools := make([]network.BackendAddressPool, 0)
|
||||||
|
if primarySLB.LoadBalancerPropertiesFormat != nil && primarySLB.BackendAddressPools != nil {
|
||||||
|
newBackendPools = *primarySLB.BackendAddressPools
|
||||||
|
}
|
||||||
|
vmSetNameToBackendIPConfigurationsToBeDeleted := make(map[string][]network.InterfaceIPConfiguration)
|
||||||
|
for j, bp := range newBackendPools {
|
||||||
|
if strings.EqualFold(to.String(bp.Name), lbBackendPoolName) {
|
||||||
|
klog.V(2).Infof("cleanBackendpoolForPrimarySLB: checking the backend pool %s from standard load balancer %s", to.String(bp.Name), to.String(primarySLB.Name))
|
||||||
|
if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil {
|
||||||
|
for i := len(*bp.BackendIPConfigurations) - 1; i >= 0; i-- {
|
||||||
|
ipConf := (*bp.BackendIPConfigurations)[i]
|
||||||
|
ipConfigID := to.String(ipConf.ID)
|
||||||
|
_, vmSetName, err := az.VMSet.GetNodeNameByIPConfigurationID(ipConfigID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
primaryVMSetName := az.VMSet.GetPrimaryVMSetName()
|
||||||
|
if !strings.EqualFold(primaryVMSetName, vmSetName) {
|
||||||
|
klog.V(2).Infof("cleanBackendpoolForPrimarySLB: found unwanted vmSet %s, decouple it from the LB", vmSetName)
|
||||||
|
// construct a backendPool that only contains the IP config of the node to be deleted
|
||||||
|
interfaceIPConfigToBeDeleted := network.InterfaceIPConfiguration{
|
||||||
|
ID: to.StringPtr(ipConfigID),
|
||||||
|
}
|
||||||
|
vmSetNameToBackendIPConfigurationsToBeDeleted[vmSetName] = append(vmSetNameToBackendIPConfigurationsToBeDeleted[vmSetName], interfaceIPConfigToBeDeleted)
|
||||||
|
*bp.BackendIPConfigurations = append((*bp.BackendIPConfigurations)[:i], (*bp.BackendIPConfigurations)[i+1:]...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
newBackendPools[j] = bp
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for vmSetName, backendIPConfigurationsToBeDeleted := range vmSetNameToBackendIPConfigurationsToBeDeleted {
|
||||||
|
backendpoolToBeDeleted := &[]network.BackendAddressPool{
|
||||||
|
{
|
||||||
|
ID: to.StringPtr(lbBackendPoolID),
|
||||||
|
BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
|
||||||
|
BackendIPConfigurations: &backendIPConfigurationsToBeDeleted,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
// decouple the backendPool from the node
|
||||||
|
err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, backendpoolToBeDeleted)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
primarySLB.BackendAddressPools = &newBackendPools
|
||||||
|
}
|
||||||
|
return primarySLB, nil
|
||||||
|
}
|
||||||
|
|
||||||
// getServiceLoadBalancer gets the loadbalancer for the service if it already exists.
|
// getServiceLoadBalancer gets the loadbalancer for the service if it already exists.
|
||||||
// If wantLb is TRUE then -it selects a new load balancer.
|
// If wantLb is TRUE then -it selects a new load balancer.
|
||||||
// In case the selected load balancer does not exist it returns network.LoadBalancer struct
|
// In case the selected load balancer does not exist it returns network.LoadBalancer struct
|
||||||
@ -284,6 +343,7 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string,
|
|||||||
var defaultLB *network.LoadBalancer
|
var defaultLB *network.LoadBalancer
|
||||||
primaryVMSetName := az.VMSet.GetPrimaryVMSetName()
|
primaryVMSetName := az.VMSet.GetPrimaryVMSetName()
|
||||||
defaultLBName := az.getAzureLoadBalancerName(clusterName, primaryVMSetName, isInternal)
|
defaultLBName := az.getAzureLoadBalancerName(clusterName, primaryVMSetName, isInternal)
|
||||||
|
useMultipleSLBs := az.useStandardLoadBalancer() && az.EnableMultipleStandardLoadBalancers
|
||||||
|
|
||||||
existingLBs, err := az.ListLB(service)
|
existingLBs, err := az.ListLB(service)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -293,6 +353,13 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string,
|
|||||||
// check if the service already has a load balancer
|
// check if the service already has a load balancer
|
||||||
for i := range existingLBs {
|
for i := range existingLBs {
|
||||||
existingLB := existingLBs[i]
|
existingLB := existingLBs[i]
|
||||||
|
if strings.EqualFold(to.String(existingLB.Name), clusterName) && useMultipleSLBs {
|
||||||
|
cleanedLB, err := az.cleanBackendpoolForPrimarySLB(&existingLB, service, clusterName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, false, err
|
||||||
|
}
|
||||||
|
existingLB = *cleanedLB
|
||||||
|
}
|
||||||
if strings.EqualFold(*existingLB.Name, defaultLBName) {
|
if strings.EqualFold(*existingLB.Name, defaultLBName) {
|
||||||
defaultLB = &existingLB
|
defaultLB = &existingLB
|
||||||
}
|
}
|
||||||
@ -312,13 +379,15 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
hasMode, _, _ := getServiceLoadBalancerMode(service)
|
hasMode, _, _ := getServiceLoadBalancerMode(service)
|
||||||
if az.useStandardLoadBalancer() && hasMode {
|
useSingleSLB := az.useStandardLoadBalancer() && !az.EnableMultipleStandardLoadBalancers
|
||||||
return nil, nil, false, fmt.Errorf("standard load balancer doesn't work with annotation %q", ServiceAnnotationLoadBalancerMode)
|
if useSingleSLB && hasMode {
|
||||||
|
klog.Warningf("single standard load balancer doesn't work with annotation %q, would ignore it", ServiceAnnotationLoadBalancerMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// service does not have a basic load balancer, select one.
|
// Service does not have a load balancer, select one.
|
||||||
// Standard load balancer doesn't need this because all backends nodes should be added to same LB.
|
// Single standard load balancer doesn't need this because
|
||||||
if wantLb && !az.useStandardLoadBalancer() {
|
// all backends nodes should be added to same LB.
|
||||||
|
if wantLb && !useSingleSLB {
|
||||||
// select new load balancer for service
|
// select new load balancer for service
|
||||||
selectedLB, exists, err := az.selectLoadBalancer(clusterName, service, &existingLBs, nodes)
|
selectedLB, exists, err := az.selectLoadBalancer(clusterName, service, &existingLBs, nodes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -358,7 +427,7 @@ func (az *Cloud) selectLoadBalancer(clusterName string, service *v1.Service, exi
|
|||||||
klog.Errorf("az.selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - az.GetVMSetNames failed, err=(%v)", clusterName, serviceName, isInternal, err)
|
klog.Errorf("az.selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - az.GetVMSetNames failed, err=(%v)", clusterName, serviceName, isInternal, err)
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
klog.Infof("selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - vmSetNames %v", clusterName, serviceName, isInternal, *vmSetNames)
|
klog.V(2).Infof("selectLoadBalancer: cluster(%s) service(%s) isInternal(%t) - vmSetNames %v", clusterName, serviceName, isInternal, *vmSetNames)
|
||||||
|
|
||||||
mapExistingLBs := map[string]network.LoadBalancer{}
|
mapExistingLBs := map[string]network.LoadBalancer{}
|
||||||
for _, lb := range *existingLBs {
|
for _, lb := range *existingLBs {
|
||||||
@ -371,9 +440,16 @@ func (az *Cloud) selectLoadBalancer(clusterName string, service *v1.Service, exi
|
|||||||
if !exists {
|
if !exists {
|
||||||
// select this LB as this is a new LB and will have minimum rules
|
// select this LB as this is a new LB and will have minimum rules
|
||||||
// create tmp lb struct to hold metadata for the new load-balancer
|
// create tmp lb struct to hold metadata for the new load-balancer
|
||||||
|
var loadBalancerSKU network.LoadBalancerSkuName
|
||||||
|
if az.useStandardLoadBalancer() {
|
||||||
|
loadBalancerSKU = network.LoadBalancerSkuNameStandard
|
||||||
|
} else {
|
||||||
|
loadBalancerSKU = network.LoadBalancerSkuNameBasic
|
||||||
|
}
|
||||||
selectedLB = &network.LoadBalancer{
|
selectedLB = &network.LoadBalancer{
|
||||||
Name: &currLBName,
|
Name: &currLBName,
|
||||||
Location: &az.Location,
|
Location: &az.Location,
|
||||||
|
Sku: &network.LoadBalancerSku{Name: loadBalancerSKU},
|
||||||
LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{},
|
LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -852,17 +928,17 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend
|
|||||||
return config.PublicIPAddress != nil && !strings.EqualFold(to.String(pip.ID), to.String(config.PublicIPAddress.ID)), nil
|
return config.PublicIPAddress != nil && !strings.EqualFold(to.String(pip.ID), to.String(config.PublicIPAddress.ID)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// isFrontendIPConfigIsUnsafeToDelete checks if a frontend IP config is safe to be deleted.
|
// isFrontendIPConfigUnsafeToDelete checks if a frontend IP config is safe to be deleted.
|
||||||
// It is safe to be deleted if and only if there is no reference from other
|
// It is safe to be deleted if and only if there is no reference from other
|
||||||
// loadBalancing resources, including loadBalancing rules, outbound rules, inbound NAT rules
|
// loadBalancing resources, including loadBalancing rules, outbound rules, inbound NAT rules
|
||||||
// and inbound NAT pools.
|
// and inbound NAT pools.
|
||||||
func (az *Cloud) isFrontendIPConfigIsUnsafeToDelete(
|
func (az *Cloud) isFrontendIPConfigUnsafeToDelete(
|
||||||
lb *network.LoadBalancer,
|
lb *network.LoadBalancer,
|
||||||
service *v1.Service,
|
service *v1.Service,
|
||||||
fipConfigID *string,
|
fipConfigID *string,
|
||||||
) (bool, error) {
|
) (bool, error) {
|
||||||
if lb == nil || fipConfigID == nil || *fipConfigID == "" {
|
if lb == nil || fipConfigID == nil || *fipConfigID == "" {
|
||||||
return false, fmt.Errorf("isFrontendIPConfigIsUnsafeToDelete: incorrect parameters")
|
return false, fmt.Errorf("isFrontendIPConfigUnsafeToDelete: incorrect parameters")
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -896,7 +972,7 @@ func (az *Cloud) isFrontendIPConfigIsUnsafeToDelete(
|
|||||||
lbRule.FrontendIPConfiguration.ID != nil &&
|
lbRule.FrontendIPConfiguration.ID != nil &&
|
||||||
strings.EqualFold(*lbRule.FrontendIPConfiguration.ID, *fipConfigID) {
|
strings.EqualFold(*lbRule.FrontendIPConfiguration.ID, *fipConfigID) {
|
||||||
if !az.serviceOwnsRule(service, *lbRule.Name) {
|
if !az.serviceOwnsRule(service, *lbRule.Name) {
|
||||||
warningMsg := fmt.Sprintf("isFrontendIPConfigIsUnsafeToDelete: frontend IP configuration with ID %s on LB %s cannot be deleted because it is being referenced by load balancing rules of other services", *fipConfigID, *lb.Name)
|
warningMsg := fmt.Sprintf("isFrontendIPConfigUnsafeToDelete: frontend IP configuration with ID %s on LB %s cannot be deleted because it is being referenced by load balancing rules of other services", *fipConfigID, *lb.Name)
|
||||||
klog.Warning(warningMsg)
|
klog.Warning(warningMsg)
|
||||||
az.Event(service, v1.EventTypeWarning, "DeletingFrontendIPConfiguration", warningMsg)
|
az.Event(service, v1.EventTypeWarning, "DeletingFrontendIPConfiguration", warningMsg)
|
||||||
unsafe = true
|
unsafe = true
|
||||||
@ -911,7 +987,7 @@ func (az *Cloud) isFrontendIPConfigIsUnsafeToDelete(
|
|||||||
if outboundRule.OutboundRulePropertiesFormat != nil && outboundRule.FrontendIPConfigurations != nil {
|
if outboundRule.OutboundRulePropertiesFormat != nil && outboundRule.FrontendIPConfigurations != nil {
|
||||||
outboundRuleFIPConfigs := *outboundRule.FrontendIPConfigurations
|
outboundRuleFIPConfigs := *outboundRule.FrontendIPConfigurations
|
||||||
if found := findMatchedOutboundRuleFIPConfig(fipConfigID, outboundRuleFIPConfigs); found {
|
if found := findMatchedOutboundRuleFIPConfig(fipConfigID, outboundRuleFIPConfigs); found {
|
||||||
warningMsg := fmt.Sprintf("isFrontendIPConfigIsUnsafeToDelete: frontend IP configuration with ID %s on LB %s cannot be deleted because it is being referenced by the outbound rule %s", *fipConfigID, *lb.Name, *outboundRule.Name)
|
warningMsg := fmt.Sprintf("isFrontendIPConfigUnsafeToDelete: frontend IP configuration with ID %s on LB %s cannot be deleted because it is being referenced by the outbound rule %s", *fipConfigID, *lb.Name, *outboundRule.Name)
|
||||||
klog.Warning(warningMsg)
|
klog.Warning(warningMsg)
|
||||||
az.Event(service, v1.EventTypeWarning, "DeletingFrontendIPConfiguration", warningMsg)
|
az.Event(service, v1.EventTypeWarning, "DeletingFrontendIPConfiguration", warningMsg)
|
||||||
unsafe = true
|
unsafe = true
|
||||||
@ -927,7 +1003,7 @@ func (az *Cloud) isFrontendIPConfigIsUnsafeToDelete(
|
|||||||
inboundNatRule.FrontendIPConfiguration != nil &&
|
inboundNatRule.FrontendIPConfiguration != nil &&
|
||||||
inboundNatRule.FrontendIPConfiguration.ID != nil &&
|
inboundNatRule.FrontendIPConfiguration.ID != nil &&
|
||||||
strings.EqualFold(*inboundNatRule.FrontendIPConfiguration.ID, *fipConfigID) {
|
strings.EqualFold(*inboundNatRule.FrontendIPConfiguration.ID, *fipConfigID) {
|
||||||
warningMsg := fmt.Sprintf("isFrontendIPConfigIsUnsafeToDelete: frontend IP configuration with ID %s on LB %s cannot be deleted because it is being referenced by the inbound NAT rule %s", *fipConfigID, *lb.Name, *inboundNatRule.Name)
|
warningMsg := fmt.Sprintf("isFrontendIPConfigUnsafeToDelete: frontend IP configuration with ID %s on LB %s cannot be deleted because it is being referenced by the inbound NAT rule %s", *fipConfigID, *lb.Name, *inboundNatRule.Name)
|
||||||
klog.Warning(warningMsg)
|
klog.Warning(warningMsg)
|
||||||
az.Event(service, v1.EventTypeWarning, "DeletingFrontendIPConfiguration", warningMsg)
|
az.Event(service, v1.EventTypeWarning, "DeletingFrontendIPConfiguration", warningMsg)
|
||||||
unsafe = true
|
unsafe = true
|
||||||
@ -942,7 +1018,7 @@ func (az *Cloud) isFrontendIPConfigIsUnsafeToDelete(
|
|||||||
inboundNatPool.FrontendIPConfiguration != nil &&
|
inboundNatPool.FrontendIPConfiguration != nil &&
|
||||||
inboundNatPool.FrontendIPConfiguration.ID != nil &&
|
inboundNatPool.FrontendIPConfiguration.ID != nil &&
|
||||||
strings.EqualFold(*inboundNatPool.FrontendIPConfiguration.ID, *fipConfigID) {
|
strings.EqualFold(*inboundNatPool.FrontendIPConfiguration.ID, *fipConfigID) {
|
||||||
warningMsg := fmt.Sprintf("isFrontendIPConfigIsUnsafeToDelete: frontend IP configuration with ID %s on LB %s cannot be deleted because it is being referenced by the inbound NAT pool %s", *fipConfigID, *lb.Name, *inboundNatPool.Name)
|
warningMsg := fmt.Sprintf("isFrontendIPConfigUnsafeToDelete: frontend IP configuration with ID %s on LB %s cannot be deleted because it is being referenced by the inbound NAT pool %s", *fipConfigID, *lb.Name, *inboundNatPool.Name)
|
||||||
klog.Warning(warningMsg)
|
klog.Warning(warningMsg)
|
||||||
az.Event(service, v1.EventTypeWarning, "DeletingFrontendIPConfiguration", warningMsg)
|
az.Event(service, v1.EventTypeWarning, "DeletingFrontendIPConfiguration", warningMsg)
|
||||||
unsafe = true
|
unsafe = true
|
||||||
@ -989,7 +1065,7 @@ func nodeNameInNodes(nodeName string, nodes []*v1.Node) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// This ensures load balancer exists and the frontend ip config is setup.
|
// reconcileLoadBalancer ensures load balancer exists and the frontend ip config is setup.
|
||||||
// This also reconciles the Service's Ports with the LoadBalancer config.
|
// This also reconciles the Service's Ports with the LoadBalancer config.
|
||||||
// This entails adding rules/probes for expected Ports and removing stale rules/ports.
|
// This entails adding rules/probes for expected Ports and removing stale rules/ports.
|
||||||
// nodes only used if wantLb is true
|
// nodes only used if wantLb is true
|
||||||
@ -1035,7 +1111,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
|
|||||||
if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil {
|
if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil {
|
||||||
for _, ipConf := range *bp.BackendIPConfigurations {
|
for _, ipConf := range *bp.BackendIPConfigurations {
|
||||||
ipConfID := to.String(ipConf.ID)
|
ipConfID := to.String(ipConf.ID)
|
||||||
nodeName, err := az.VMSet.GetNodeNameByIPConfigurationID(ipConfID)
|
nodeName, _, err := az.VMSet.GetNodeNameByIPConfigurationID(ipConfID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -1106,7 +1182,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if isServiceOwnsFrontendIP {
|
if isServiceOwnsFrontendIP {
|
||||||
unsafe, err := az.isFrontendIPConfigIsUnsafeToDelete(lb, service, config.ID)
|
unsafe, err := az.isFrontendIPConfigUnsafeToDelete(lb, service, config.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -37,10 +37,12 @@ import (
|
|||||||
|
|
||||||
v1 "k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient"
|
||||||
"k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient/mockloadbalancerclient"
|
"k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient/mockloadbalancerclient"
|
||||||
"k8s.io/legacy-cloud-providers/azure/clients/publicipclient/mockpublicipclient"
|
"k8s.io/legacy-cloud-providers/azure/clients/publicipclient/mockpublicipclient"
|
||||||
"k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient/mocksecuritygroupclient"
|
"k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient/mocksecuritygroupclient"
|
||||||
"k8s.io/legacy-cloud-providers/azure/clients/subnetclient/mocksubnetclient"
|
"k8s.io/legacy-cloud-providers/azure/clients/subnetclient/mocksubnetclient"
|
||||||
|
"k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient"
|
||||||
"k8s.io/legacy-cloud-providers/azure/retry"
|
"k8s.io/legacy-cloud-providers/azure/retry"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -1169,14 +1171,6 @@ func TestGetServiceLoadBalancer(t *testing.T) {
|
|||||||
expectedExists: true,
|
expectedExists: true,
|
||||||
expectedError: false,
|
expectedError: false,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
desc: "getServiceLoadBalancer shall report error if there are loadbalancer mode annotations on a standard lb",
|
|
||||||
service: getTestService("service1", v1.ProtocolTCP, nil, false, 80),
|
|
||||||
annotations: map[string]string{ServiceAnnotationLoadBalancerMode: "__auto__"},
|
|
||||||
sku: "standard",
|
|
||||||
expectedExists: false,
|
|
||||||
expectedError: true,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
desc: "getServiceLoadBalancer shall select the lb with minimum lb rules if wantLb is true, the sku is " +
|
desc: "getServiceLoadBalancer shall select the lb with minimum lb rules if wantLb is true, the sku is " +
|
||||||
"not standard and there are existing lbs already",
|
"not standard and there are existing lbs already",
|
||||||
@ -3310,7 +3304,7 @@ func TestIsFrontendIPConfigIsUnsafeToDelete(t *testing.T) {
|
|||||||
unsafe bool
|
unsafe bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
desc: "isFrontendIPConfigIsUnsafeToDelete should return true if there is a " +
|
desc: "isFrontendIPConfigUnsafeToDelete should return true if there is a " +
|
||||||
"loadBalancing rule from other service referencing the frontend IP config",
|
"loadBalancing rule from other service referencing the frontend IP config",
|
||||||
existingLB: &network.LoadBalancer{
|
existingLB: &network.LoadBalancer{
|
||||||
Name: to.StringPtr("lb"),
|
Name: to.StringPtr("lb"),
|
||||||
@ -3328,7 +3322,7 @@ func TestIsFrontendIPConfigIsUnsafeToDelete(t *testing.T) {
|
|||||||
unsafe: true,
|
unsafe: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
desc: "isFrontendIPConfigIsUnsafeToDelete should return false if there is a " +
|
desc: "isFrontendIPConfigUnsafeToDelete should return false if there is a " +
|
||||||
"loadBalancing rule from this service referencing the frontend IP config",
|
"loadBalancing rule from this service referencing the frontend IP config",
|
||||||
existingLB: &network.LoadBalancer{
|
existingLB: &network.LoadBalancer{
|
||||||
Name: to.StringPtr("lb"),
|
Name: to.StringPtr("lb"),
|
||||||
@ -3348,7 +3342,7 @@ func TestIsFrontendIPConfigIsUnsafeToDelete(t *testing.T) {
|
|||||||
unsafe: true,
|
unsafe: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
desc: "isFrontendIPConfigIsUnsafeToDelete should return false if there is a " +
|
desc: "isFrontendIPConfigUnsafeToDelete should return false if there is a " +
|
||||||
"outbound rule referencing the frontend IP config",
|
"outbound rule referencing the frontend IP config",
|
||||||
existingLB: &network.LoadBalancer{
|
existingLB: &network.LoadBalancer{
|
||||||
Name: to.StringPtr("lb"),
|
Name: to.StringPtr("lb"),
|
||||||
@ -3365,7 +3359,7 @@ func TestIsFrontendIPConfigIsUnsafeToDelete(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
desc: "isFrontendIPConfigIsUnsafeToDelete should return true if there is a " +
|
desc: "isFrontendIPConfigUnsafeToDelete should return true if there is a " +
|
||||||
"inbound NAT rule referencing the frontend IP config",
|
"inbound NAT rule referencing the frontend IP config",
|
||||||
existingLB: &network.LoadBalancer{
|
existingLB: &network.LoadBalancer{
|
||||||
Name: to.StringPtr("lb"),
|
Name: to.StringPtr("lb"),
|
||||||
@ -3383,7 +3377,7 @@ func TestIsFrontendIPConfigIsUnsafeToDelete(t *testing.T) {
|
|||||||
unsafe: true,
|
unsafe: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
desc: "isFrontendIPConfigIsUnsafeToDelete should return true if there is a " +
|
desc: "isFrontendIPConfigUnsafeToDelete should return true if there is a " +
|
||||||
"inbound NAT pool referencing the frontend IP config",
|
"inbound NAT pool referencing the frontend IP config",
|
||||||
existingLB: &network.LoadBalancer{
|
existingLB: &network.LoadBalancer{
|
||||||
Name: to.StringPtr("lb"),
|
Name: to.StringPtr("lb"),
|
||||||
@ -3403,7 +3397,7 @@ func TestIsFrontendIPConfigIsUnsafeToDelete(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, testCase := range testCases {
|
for _, testCase := range testCases {
|
||||||
unsafe, _ := az.isFrontendIPConfigIsUnsafeToDelete(testCase.existingLB, &service, fipID)
|
unsafe, _ := az.isFrontendIPConfigUnsafeToDelete(testCase.existingLB, &service, fipID)
|
||||||
assert.Equal(t, testCase.unsafe, unsafe, testCase.desc)
|
assert.Equal(t, testCase.unsafe, unsafe, testCase.desc)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -3529,3 +3523,71 @@ func TestCheckLoadBalancerResourcesConflicted(t *testing.T) {
|
|||||||
assert.Equal(t, testCase.expectedErr, err != nil, testCase.desc)
|
assert.Equal(t, testCase.expectedErr, err != nil, testCase.desc)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCleanBackendpoolForPrimarySLB(t *testing.T) {
|
||||||
|
ctrl := gomock.NewController(t)
|
||||||
|
defer ctrl.Finish()
|
||||||
|
cloud := GetTestCloud(ctrl)
|
||||||
|
cloud.LoadBalancerSku = loadBalancerSkuStandard
|
||||||
|
cloud.EnableMultipleStandardLoadBalancers = true
|
||||||
|
cloud.PrimaryAvailabilitySetName = "agentpool1-availabilitySet-00000000"
|
||||||
|
clusterName := "testCluster"
|
||||||
|
service := getTestService("test", v1.ProtocolTCP, nil, false, 80)
|
||||||
|
lb := buildDefaultTestLB("testCluster", []string{
|
||||||
|
"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1",
|
||||||
|
"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool2-00000000-nic-1/ipConfigurations/ipconfig1",
|
||||||
|
})
|
||||||
|
existingVMForAS1 := buildDefaultTestVirtualMachine("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/agentpool1-availabilitySet-00000000", []string{"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1"})
|
||||||
|
existingVMForAS2 := buildDefaultTestVirtualMachine("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/agentpool2-availabilitySet-00000000", []string{"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool2-00000000-nic-1"})
|
||||||
|
existingNIC := buildDefaultTestInterface(true, []string{"/subscriptions/sub/resourceGroups/gh/providers/Microsoft.Network/loadBalancers/testCluster/backendAddressPools/testCluster"})
|
||||||
|
mockVMClient := mockvmclient.NewMockInterface(ctrl)
|
||||||
|
mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "k8s-agentpool1-00000000-1", gomock.Any()).Return(existingVMForAS1, nil)
|
||||||
|
mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "k8s-agentpool2-00000000-1", gomock.Any()).Return(existingVMForAS2, nil)
|
||||||
|
cloud.VirtualMachinesClient = mockVMClient
|
||||||
|
mockNICClient := mockinterfaceclient.NewMockInterface(ctrl)
|
||||||
|
mockNICClient.EXPECT().Get(gomock.Any(), "rg", "k8s-agentpool2-00000000-nic-1", gomock.Any()).Return(existingNIC, nil)
|
||||||
|
mockNICClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
|
||||||
|
cloud.InterfacesClient = mockNICClient
|
||||||
|
cleanedLB, err := cloud.cleanBackendpoolForPrimarySLB(&lb, &service, clusterName)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
expectedLB := network.LoadBalancer{
|
||||||
|
Name: to.StringPtr("testCluster"),
|
||||||
|
LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
|
||||||
|
BackendAddressPools: &[]network.BackendAddressPool{
|
||||||
|
{
|
||||||
|
Name: to.StringPtr("testCluster"),
|
||||||
|
BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
|
||||||
|
BackendIPConfigurations: &[]network.InterfaceIPConfiguration{
|
||||||
|
{
|
||||||
|
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
assert.Equal(t, expectedLB, *cleanedLB)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildDefaultTestLB(name string, backendIPConfigs []string) network.LoadBalancer {
|
||||||
|
expectedLB := network.LoadBalancer{
|
||||||
|
Name: to.StringPtr(name),
|
||||||
|
LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
|
||||||
|
BackendAddressPools: &[]network.BackendAddressPool{
|
||||||
|
{
|
||||||
|
Name: to.StringPtr(name),
|
||||||
|
BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
|
||||||
|
BackendIPConfigurations: &[]network.InterfaceIPConfiguration{},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
backendIPConfigurations := make([]network.InterfaceIPConfiguration, 0)
|
||||||
|
for _, ipConfig := range backendIPConfigs {
|
||||||
|
backendIPConfigurations = append(backendIPConfigurations, network.InterfaceIPConfiguration{ID: to.StringPtr(ipConfig)})
|
||||||
|
}
|
||||||
|
(*expectedLB.BackendAddressPools)[0].BackendIPConfigurations = &backendIPConfigurations
|
||||||
|
return expectedLB
|
||||||
|
}
|
||||||
|
@ -154,7 +154,11 @@ func (az *Cloud) getAzureLoadBalancerName(clusterName string, vmSetName string,
|
|||||||
clusterName = az.LoadBalancerName
|
clusterName = az.LoadBalancerName
|
||||||
}
|
}
|
||||||
lbNamePrefix := vmSetName
|
lbNamePrefix := vmSetName
|
||||||
if strings.EqualFold(vmSetName, az.VMSet.GetPrimaryVMSetName()) || az.useStandardLoadBalancer() {
|
// The LB name prefix is set to the name of the cluster when:
|
||||||
|
// 1. the LB belongs to the primary agent pool.
|
||||||
|
// 2. using the single SLB;
|
||||||
|
useSingleSLB := az.useStandardLoadBalancer() && !az.EnableMultipleStandardLoadBalancers
|
||||||
|
if strings.EqualFold(vmSetName, az.VMSet.GetPrimaryVMSetName()) || useSingleSLB {
|
||||||
lbNamePrefix = clusterName
|
lbNamePrefix = clusterName
|
||||||
}
|
}
|
||||||
if isInternal {
|
if isInternal {
|
||||||
@ -215,7 +219,7 @@ func getPrimaryInterfaceID(machine compute.VirtualMachine) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, ref := range *machine.NetworkProfile.NetworkInterfaces {
|
for _, ref := range *machine.NetworkProfile.NetworkInterfaces {
|
||||||
if *ref.Primary {
|
if to.Bool(ref.Primary) {
|
||||||
return *ref.ID, nil
|
return *ref.ID, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -258,7 +262,7 @@ func getIPConfigByIPFamily(nic network.Interface, IPv6 bool) (*network.Interface
|
|||||||
return &ref, nil
|
return &ref, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("failed to determine the ipconfig(IPv6=%v). nicname=%q", IPv6, *nic.Name)
|
return nil, fmt.Errorf("failed to determine the ipconfig(IPv6=%v). nicname=%q", IPv6, to.String(nic.Name))
|
||||||
}
|
}
|
||||||
|
|
||||||
func isInternalLoadBalancer(lb *network.LoadBalancer) bool {
|
func isInternalLoadBalancer(lb *network.LoadBalancer) bool {
|
||||||
@ -658,11 +662,14 @@ func (as *availabilitySet) getAgentPoolAvailabilitySets(nodes []*v1.Node) (agent
|
|||||||
// GetVMSetNames selects all possible availability sets or scale sets
|
// GetVMSetNames selects all possible availability sets or scale sets
|
||||||
// (depending vmType configured) for service load balancer, if the service has
|
// (depending vmType configured) for service load balancer, if the service has
|
||||||
// no loadbalancer mode annotation returns the primary VMSet. If service annotation
|
// no loadbalancer mode annotation returns the primary VMSet. If service annotation
|
||||||
// for loadbalancer exists then return the eligible VMSet.
|
// for loadbalancer exists then returns the eligible VMSet. The mode selection
|
||||||
|
// annotation would be ignored when using one SLB per cluster.
|
||||||
func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) (availabilitySetNames *[]string, err error) {
|
func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) (availabilitySetNames *[]string, err error) {
|
||||||
hasMode, isAuto, serviceAvailabilitySetNames := getServiceLoadBalancerMode(service)
|
hasMode, isAuto, serviceAvailabilitySetNames := getServiceLoadBalancerMode(service)
|
||||||
if !hasMode {
|
useSingleSLB := as.useStandardLoadBalancer() && !as.EnableMultipleStandardLoadBalancers
|
||||||
// no mode specified in service annotation default to PrimaryAvailabilitySetName
|
if !hasMode || useSingleSLB {
|
||||||
|
// no mode specified in service annotation or use single SLB mode
|
||||||
|
// default to PrimaryAvailabilitySetName
|
||||||
availabilitySetNames = &[]string{as.Config.PrimaryAvailabilitySetName}
|
availabilitySetNames = &[]string{as.Config.PrimaryAvailabilitySetName}
|
||||||
return availabilitySetNames, nil
|
return availabilitySetNames, nil
|
||||||
}
|
}
|
||||||
@ -746,11 +753,20 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
|
|||||||
// Node's real availability set name:
|
// Node's real availability set name:
|
||||||
// - For basic SKU load balancer, errNotInVMSet should be returned if the node's
|
// - For basic SKU load balancer, errNotInVMSet should be returned if the node's
|
||||||
// availability set is mismatched with vmSetName.
|
// availability set is mismatched with vmSetName.
|
||||||
// - For standard SKU load balancer, backend could belong to multiple VMAS, so we
|
// - For single standard SKU load balancer, backend could belong to multiple VMAS, so we
|
||||||
// don't check vmSet for it.
|
// don't check vmSet for it.
|
||||||
if vmSetName != "" && !as.useStandardLoadBalancer() {
|
// - For multiple standard SKU load balancers, the behavior is similar to the basic LB.
|
||||||
expectedAvailabilitySetName := as.getAvailabilitySetID(nodeResourceGroup, vmSetName)
|
needCheck := false
|
||||||
if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) {
|
if !as.useStandardLoadBalancer() {
|
||||||
|
// need to check the vmSet name when using the basic LB
|
||||||
|
needCheck = true
|
||||||
|
} else if as.EnableMultipleStandardLoadBalancers {
|
||||||
|
// need to check the vmSet name when using multiple standard LBs
|
||||||
|
needCheck = true
|
||||||
|
}
|
||||||
|
if vmSetName != "" && needCheck {
|
||||||
|
expectedAvailabilitySetID := as.getAvailabilitySetID(nodeResourceGroup, vmSetName)
|
||||||
|
if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetID) {
|
||||||
klog.V(3).Infof(
|
klog.V(3).Infof(
|
||||||
"GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName)
|
"GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName)
|
||||||
return network.Interface{}, "", errNotInVMSet
|
return network.Interface{}, "", errNotInVMSet
|
||||||
@ -933,7 +949,7 @@ func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backend
|
|||||||
errors := make([]error, 0)
|
errors := make([]error, 0)
|
||||||
for i := range ipConfigurationIDs {
|
for i := range ipConfigurationIDs {
|
||||||
ipConfigurationID := ipConfigurationIDs[i]
|
ipConfigurationID := ipConfigurationIDs[i]
|
||||||
nodeName, err := as.GetNodeNameByIPConfigurationID(ipConfigurationID)
|
nodeName, _, err := as.GetNodeNameByIPConfigurationID(ipConfigurationID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
|
klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
|
||||||
errors = append(errors, err)
|
errors = append(errors, err)
|
||||||
@ -951,11 +967,10 @@ func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backend
|
|||||||
klog.Errorf("error: az.EnsureBackendPoolDeleted(%s), az.VMSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err)
|
klog.Errorf("error: az.EnsureBackendPoolDeleted(%s), az.VMSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
matches := vmasIDRE.FindStringSubmatch(vmasID)
|
vmasName, err := getAvailabilitySetNameByID(vmasID)
|
||||||
if len(matches) != 2 {
|
if err != nil {
|
||||||
return fmt.Errorf("EnsureBackendPoolDeleted: failed to parse the VMAS ID %s: %v", vmasID, err)
|
return fmt.Errorf("EnsureBackendPoolDeleted: failed to parse the VMAS ID %s: %v", vmasID, err)
|
||||||
}
|
}
|
||||||
vmasName := matches[1]
|
|
||||||
// Only remove nodes belonging to specified vmSet to basic LB backends.
|
// Only remove nodes belonging to specified vmSet to basic LB backends.
|
||||||
if !strings.EqualFold(vmasName, vmSetName) {
|
if !strings.EqualFold(vmasName, vmSetName) {
|
||||||
klog.V(2).Infof("EnsureBackendPoolDeleted: skipping the node %s belonging to another vm set %s", nodeName, vmasName)
|
klog.V(2).Infof("EnsureBackendPoolDeleted: skipping the node %s belonging to another vm set %s", nodeName, vmasName)
|
||||||
@ -976,7 +991,8 @@ func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backend
|
|||||||
// found primary ip configuration
|
// found primary ip configuration
|
||||||
if ipConf.LoadBalancerBackendAddressPools != nil {
|
if ipConf.LoadBalancerBackendAddressPools != nil {
|
||||||
newLBAddressPools := *ipConf.LoadBalancerBackendAddressPools
|
newLBAddressPools := *ipConf.LoadBalancerBackendAddressPools
|
||||||
for k, pool := range newLBAddressPools {
|
for k := len(newLBAddressPools) - 1; k >= 0; k-- {
|
||||||
|
pool := newLBAddressPools[k]
|
||||||
if strings.EqualFold(to.String(pool.ID), backendPoolID) {
|
if strings.EqualFold(to.String(pool.ID), backendPoolID) {
|
||||||
newLBAddressPools = append(newLBAddressPools[:k], newLBAddressPools[k+1:]...)
|
newLBAddressPools = append(newLBAddressPools[:k], newLBAddressPools[k+1:]...)
|
||||||
break
|
break
|
||||||
@ -1012,6 +1028,15 @@ func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backend
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getAvailabilitySetNameByID(asID string) (string, error) {
|
||||||
|
matches := vmasIDRE.FindStringSubmatch(asID)
|
||||||
|
if len(matches) != 2 {
|
||||||
|
return "", fmt.Errorf("getAvailabilitySetNameByID: failed to parse the VMAS ID %s", asID)
|
||||||
|
}
|
||||||
|
vmasName := matches[1]
|
||||||
|
return vmasName, nil
|
||||||
|
}
|
||||||
|
|
||||||
// get a storage account by UUID
|
// get a storage account by UUID
|
||||||
func generateStorageAccountName(accountNamePrefix string) string {
|
func generateStorageAccountName(accountNamePrefix string) string {
|
||||||
uniqueID := strings.Replace(string(uuid.NewUUID()), "-", "", -1)
|
uniqueID := strings.Replace(string(uuid.NewUUID()), "-", "", -1)
|
||||||
@ -1022,15 +1047,32 @@ func generateStorageAccountName(accountNamePrefix string) string {
|
|||||||
return accountName
|
return accountName
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
|
// GetNodeNameByIPConfigurationID gets the node name and the availabilitySet name by IP configuration ID.
|
||||||
func (as *availabilitySet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
|
func (as *availabilitySet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, string, error) {
|
||||||
matches := nicIDRE.FindStringSubmatch(ipConfigurationID)
|
matches := nicIDRE.FindStringSubmatch(ipConfigurationID)
|
||||||
if len(matches) != 3 {
|
if len(matches) != 3 {
|
||||||
klog.V(4).Infof("Can not extract VM name from ipConfigurationID (%s)", ipConfigurationID)
|
klog.V(4).Infof("Can not extract VM name from ipConfigurationID (%s)", ipConfigurationID)
|
||||||
return "", fmt.Errorf("invalid ip config ID %s", ipConfigurationID)
|
return "", "", fmt.Errorf("invalid ip config ID %s", ipConfigurationID)
|
||||||
}
|
}
|
||||||
|
|
||||||
prefix := matches[1]
|
prefix := matches[1]
|
||||||
suffix := matches[2]
|
suffix := matches[2]
|
||||||
return fmt.Sprintf("%s-%s", prefix, suffix), nil
|
nodeName := fmt.Sprintf("%s-%s", prefix, suffix)
|
||||||
|
|
||||||
|
vm, err := as.getVirtualMachine(types.NodeName(nodeName), azcache.CacheReadTypeDefault)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("cannot get the virtual machine by node name %s", nodeName)
|
||||||
|
}
|
||||||
|
asID := ""
|
||||||
|
if vm.VirtualMachineProperties != nil && vm.AvailabilitySet != nil {
|
||||||
|
asID = to.String(vm.AvailabilitySet.ID)
|
||||||
|
}
|
||||||
|
if asID == "" {
|
||||||
|
return "", "", fmt.Errorf("cannot get the availability set ID from the virtual machine with node name %s", nodeName)
|
||||||
|
}
|
||||||
|
asName, err := getAvailabilitySetNameByID(asID)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("cannot get the availability set name by the availability set ID %s", asID)
|
||||||
|
}
|
||||||
|
return nodeName, strings.ToLower(asName), nil
|
||||||
}
|
}
|
||||||
|
@ -583,19 +583,8 @@ func TestGetStandardVMPrimaryInterfaceID(t *testing.T) {
|
|||||||
expectedErrMsg error
|
expectedErrMsg error
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "GetPrimaryInterfaceID should get the only NIC ID",
|
name: "GetPrimaryInterfaceID should get the only NIC ID",
|
||||||
vm: compute.VirtualMachine{
|
vm: buildDefaultTestVirtualMachine("", []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic"}),
|
||||||
Name: to.StringPtr("vm1"),
|
|
||||||
VirtualMachineProperties: &compute.VirtualMachineProperties{
|
|
||||||
NetworkProfile: &compute.NetworkProfile{
|
|
||||||
NetworkInterfaces: &[]compute.NetworkInterfaceReference{
|
|
||||||
{
|
|
||||||
ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expectedNicID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic",
|
expectedNicID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -1074,7 +1063,6 @@ func TestGetStandardVMZoneByNodeName(t *testing.T) {
|
|||||||
func TestGetStandardVMSetNames(t *testing.T) {
|
func TestGetStandardVMSetNames(t *testing.T) {
|
||||||
ctrl := gomock.NewController(t)
|
ctrl := gomock.NewController(t)
|
||||||
defer ctrl.Finish()
|
defer ctrl.Finish()
|
||||||
cloud := GetTestCloud(ctrl)
|
|
||||||
|
|
||||||
asID := "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/myAvailabilitySet"
|
asID := "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/myAvailabilitySet"
|
||||||
testVM := compute.VirtualMachine{
|
testVM := compute.VirtualMachine{
|
||||||
@ -1092,6 +1080,7 @@ func TestGetStandardVMSetNames(t *testing.T) {
|
|||||||
vm []compute.VirtualMachine
|
vm []compute.VirtualMachine
|
||||||
service *v1.Service
|
service *v1.Service
|
||||||
nodes []*v1.Node
|
nodes []*v1.Node
|
||||||
|
usingSingleSLBS bool
|
||||||
expectedVMSetNames *[]string
|
expectedVMSetNames *[]string
|
||||||
expectedErrMsg error
|
expectedErrMsg error
|
||||||
}{
|
}{
|
||||||
@ -1101,6 +1090,15 @@ func TestGetStandardVMSetNames(t *testing.T) {
|
|||||||
service: &v1.Service{},
|
service: &v1.Service{},
|
||||||
expectedVMSetNames: &[]string{"as"},
|
expectedVMSetNames: &[]string{"as"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "GetVMSetNames should return the primary vm set name when using the single SLB",
|
||||||
|
vm: []compute.VirtualMachine{testVM},
|
||||||
|
service: &v1.Service{
|
||||||
|
ObjectMeta: meta.ObjectMeta{Annotations: map[string]string{ServiceAnnotationLoadBalancerMode: ServiceAnnotationLoadBalancerAutoModeValue}},
|
||||||
|
},
|
||||||
|
usingSingleSLBS: true,
|
||||||
|
expectedVMSetNames: &[]string{"as"},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "GetVMSetNames should return the correct as names if the service has auto mode annotation",
|
name: "GetVMSetNames should return the correct as names if the service has auto mode annotation",
|
||||||
vm: []compute.VirtualMachine{testVM},
|
vm: []compute.VirtualMachine{testVM},
|
||||||
@ -1164,6 +1162,11 @@ func TestGetStandardVMSetNames(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range testCases {
|
for _, test := range testCases {
|
||||||
|
cloud := GetTestCloud(ctrl)
|
||||||
|
if test.usingSingleSLBS {
|
||||||
|
cloud.EnableMultipleStandardLoadBalancers = false
|
||||||
|
cloud.LoadBalancerSku = loadBalancerSkuStandard
|
||||||
|
}
|
||||||
mockVMClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
|
mockVMClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
|
||||||
mockVMClient.EXPECT().List(gomock.Any(), cloud.ResourceGroup).Return(test.vm, nil).AnyTimes()
|
mockVMClient.EXPECT().List(gomock.Any(), cloud.ResourceGroup).Return(test.vm, nil).AnyTimes()
|
||||||
|
|
||||||
@ -1217,6 +1220,7 @@ func TestStandardEnsureHostInPool(t *testing.T) {
|
|||||||
vmSetName string
|
vmSetName string
|
||||||
nicProvisionState string
|
nicProvisionState string
|
||||||
isStandardLB bool
|
isStandardLB bool
|
||||||
|
useMultipleSLBs bool
|
||||||
expectedErrMsg error
|
expectedErrMsg error
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@ -1227,6 +1231,16 @@ func TestStandardEnsureHostInPool(t *testing.T) {
|
|||||||
nicID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic1",
|
nicID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic1",
|
||||||
vmSetName: "availabilityset-1",
|
vmSetName: "availabilityset-1",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "EnsureHostInPool should return nil if node is not in VMSet when using multiple SLBs",
|
||||||
|
service: &v1.Service{},
|
||||||
|
nodeName: "vm1",
|
||||||
|
nicName: "nic1",
|
||||||
|
nicID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic1",
|
||||||
|
vmSetName: "availabilityset-1",
|
||||||
|
isStandardLB: true,
|
||||||
|
useMultipleSLBs: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "EnsureHostInPool should report error if last segment of nicID is nil",
|
name: "EnsureHostInPool should report error if last segment of nicID is nil",
|
||||||
service: &v1.Service{},
|
service: &v1.Service{},
|
||||||
@ -1300,39 +1314,17 @@ func TestStandardEnsureHostInPool(t *testing.T) {
|
|||||||
cloud.Config.LoadBalancerSku = loadBalancerSkuStandard
|
cloud.Config.LoadBalancerSku = loadBalancerSkuStandard
|
||||||
}
|
}
|
||||||
|
|
||||||
testVM := compute.VirtualMachine{
|
if test.useMultipleSLBs {
|
||||||
Name: to.StringPtr(string(test.nodeName)),
|
cloud.EnableMultipleStandardLoadBalancers = true
|
||||||
VirtualMachineProperties: &compute.VirtualMachineProperties{
|
|
||||||
AvailabilitySet: &compute.SubResource{ID: to.StringPtr(availabilitySetID)},
|
|
||||||
NetworkProfile: &compute.NetworkProfile{
|
|
||||||
NetworkInterfaces: &[]compute.NetworkInterfaceReference{
|
|
||||||
{
|
|
||||||
ID: to.StringPtr(test.nicID),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
testNIC := network.Interface{
|
|
||||||
Name: to.StringPtr(test.nicName),
|
|
||||||
ID: to.StringPtr(test.nicID),
|
|
||||||
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
|
|
||||||
ProvisioningState: to.StringPtr(test.nicProvisionState),
|
|
||||||
IPConfigurations: &[]network.InterfaceIPConfiguration{
|
|
||||||
{
|
|
||||||
Name: to.StringPtr("ifconfig1"),
|
|
||||||
InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
|
|
||||||
LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
|
|
||||||
{
|
|
||||||
ID: to.StringPtr(backendAddressPoolID),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
testVM := buildDefaultTestVirtualMachine(availabilitySetID, []string{test.nicID})
|
||||||
|
testVM.Name = to.StringPtr(string(test.nodeName))
|
||||||
|
testNIC := buildDefaultTestInterface(false, []string{backendAddressPoolID})
|
||||||
|
testNIC.Name = to.StringPtr(test.nicName)
|
||||||
|
testNIC.ID = to.StringPtr(test.nicID)
|
||||||
|
testNIC.ProvisioningState = to.StringPtr(test.nicProvisionState)
|
||||||
|
|
||||||
mockVMClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
|
mockVMClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
|
||||||
mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, string(test.nodeName), gomock.Any()).Return(testVM, nil).AnyTimes()
|
mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, string(test.nodeName), gomock.Any()).Return(testVM, nil).AnyTimes()
|
||||||
|
|
||||||
@ -1463,37 +1455,10 @@ func TestStandardEnsureHostsInPool(t *testing.T) {
|
|||||||
cloud.Config.LoadBalancerSku = loadBalancerSkuStandard
|
cloud.Config.LoadBalancerSku = loadBalancerSkuStandard
|
||||||
cloud.Config.ExcludeMasterFromStandardLB = to.BoolPtr(true)
|
cloud.Config.ExcludeMasterFromStandardLB = to.BoolPtr(true)
|
||||||
|
|
||||||
testVM := compute.VirtualMachine{
|
testVM := buildDefaultTestVirtualMachine(availabilitySetID, []string{test.nicID})
|
||||||
Name: to.StringPtr(string(test.nodeName)),
|
testNIC := buildDefaultTestInterface(false, []string{backendAddressPoolID})
|
||||||
VirtualMachineProperties: &compute.VirtualMachineProperties{
|
testNIC.Name = to.StringPtr(test.nicName)
|
||||||
AvailabilitySet: &compute.SubResource{ID: to.StringPtr(availabilitySetID)},
|
testNIC.ID = to.StringPtr(test.nicID)
|
||||||
NetworkProfile: &compute.NetworkProfile{
|
|
||||||
NetworkInterfaces: &[]compute.NetworkInterfaceReference{
|
|
||||||
{
|
|
||||||
ID: to.StringPtr(test.nicID),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
testNIC := network.Interface{
|
|
||||||
Name: to.StringPtr(test.nicName),
|
|
||||||
ID: to.StringPtr(test.nicID),
|
|
||||||
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
|
|
||||||
IPConfigurations: &[]network.InterfaceIPConfiguration{
|
|
||||||
{
|
|
||||||
Name: to.StringPtr("ifconfig1"),
|
|
||||||
InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
|
|
||||||
LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
|
|
||||||
{
|
|
||||||
ID: to.StringPtr(backendAddressPoolID),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
mockVMClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
|
mockVMClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
|
||||||
mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, test.nodeName, gomock.Any()).Return(testVM, nil).AnyTimes()
|
mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, test.nodeName, gomock.Any()).Return(testVM, nil).AnyTimes()
|
||||||
@ -1682,7 +1647,7 @@ func TestStandardEnsureBackendPoolDeleted(t *testing.T) {
|
|||||||
existingNIC network.Interface
|
existingNIC network.Interface
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
desc: "",
|
desc: "EnsureBackendPoolDeleted should decouple the nic and the load balancer properly",
|
||||||
backendAddressPools: &[]network.BackendAddressPool{
|
backendAddressPools: &[]network.BackendAddressPool{
|
||||||
{
|
{
|
||||||
ID: to.StringPtr(backendPoolID),
|
ID: to.StringPtr(backendPoolID),
|
||||||
@ -1695,37 +1660,10 @@ func TestStandardEnsureBackendPoolDeleted(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
existingVM: compute.VirtualMachine{
|
existingVM: buildDefaultTestVirtualMachine("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/as", []string{
|
||||||
VirtualMachineProperties: &compute.VirtualMachineProperties{
|
"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1",
|
||||||
AvailabilitySet: &compute.SubResource{
|
}),
|
||||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/as"),
|
existingNIC: buildDefaultTestInterface(true, []string{"/subscriptions/sub/resourceGroups/gh/providers/Microsoft.Network/loadBalancers/testCluster/backendAddressPools/testCluster"}),
|
||||||
},
|
|
||||||
NetworkProfile: &compute.NetworkProfile{
|
|
||||||
NetworkInterfaces: &[]compute.NetworkInterfaceReference{
|
|
||||||
{
|
|
||||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
existingNIC: network.Interface{
|
|
||||||
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
|
|
||||||
ProvisioningState: to.StringPtr("Succeeded"),
|
|
||||||
IPConfigurations: &[]network.InterfaceIPConfiguration{
|
|
||||||
{
|
|
||||||
InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
|
|
||||||
Primary: to.BoolPtr(true),
|
|
||||||
LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
|
|
||||||
{
|
|
||||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1743,3 +1681,59 @@ func TestStandardEnsureBackendPoolDeleted(t *testing.T) {
|
|||||||
assert.NoError(t, err, test.desc)
|
assert.NoError(t, err, test.desc)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildDefaultTestInterface(isPrimary bool, lbBackendpoolIDs []string) network.Interface {
|
||||||
|
expectedNIC := network.Interface{
|
||||||
|
InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
|
||||||
|
ProvisioningState: to.StringPtr("Succeeded"),
|
||||||
|
IPConfigurations: &[]network.InterfaceIPConfiguration{
|
||||||
|
{
|
||||||
|
InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
|
||||||
|
Primary: to.BoolPtr(isPrimary),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
backendAddressPool := make([]network.BackendAddressPool, 0)
|
||||||
|
for _, id := range lbBackendpoolIDs {
|
||||||
|
backendAddressPool = append(backendAddressPool, network.BackendAddressPool{
|
||||||
|
ID: to.StringPtr(id),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
(*expectedNIC.IPConfigurations)[0].LoadBalancerBackendAddressPools = &backendAddressPool
|
||||||
|
return expectedNIC
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildDefaultTestVirtualMachine(asID string, nicIDs []string) compute.VirtualMachine {
|
||||||
|
expectedVM := compute.VirtualMachine{
|
||||||
|
VirtualMachineProperties: &compute.VirtualMachineProperties{
|
||||||
|
AvailabilitySet: &compute.SubResource{
|
||||||
|
ID: to.StringPtr(asID),
|
||||||
|
},
|
||||||
|
NetworkProfile: &compute.NetworkProfile{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
networkInterfaces := make([]compute.NetworkInterfaceReference, 0)
|
||||||
|
for _, nicID := range nicIDs {
|
||||||
|
networkInterfaces = append(networkInterfaces, compute.NetworkInterfaceReference{
|
||||||
|
ID: to.StringPtr(nicID),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
expectedVM.VirtualMachineProperties.NetworkProfile.NetworkInterfaces = &networkInterfaces
|
||||||
|
return expectedVM
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStandardGetNodeNameByIPConfigurationID(t *testing.T) {
|
||||||
|
ctrl := gomock.NewController(t)
|
||||||
|
defer ctrl.Finish()
|
||||||
|
cloud := GetTestCloud(ctrl)
|
||||||
|
expectedVM := buildDefaultTestVirtualMachine("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/AGENTPOOL1-AVAILABILITYSET-00000000", []string{})
|
||||||
|
mockVMClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
|
||||||
|
mockVMClient.EXPECT().Get(gomock.Any(), "rg", "k8s-agentpool1-00000000-0", gomock.Any()).Return(expectedVM, nil)
|
||||||
|
ipConfigurationID := `/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-0/ipConfigurations/ipconfig1`
|
||||||
|
nodeName, asName, err := cloud.VMSet.GetNodeNameByIPConfigurationID(ipConfigurationID)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, nodeName, "k8s-agentpool1-00000000-0")
|
||||||
|
assert.Equal(t, asName, "agentpool1-availabilityset-00000000")
|
||||||
|
}
|
||||||
|
@ -79,6 +79,6 @@ type VMSet interface {
|
|||||||
// GetPrivateIPsByNodeName returns a slice of all private ips assigned to node (ipv6 and ipv4)
|
// GetPrivateIPsByNodeName returns a slice of all private ips assigned to node (ipv6 and ipv4)
|
||||||
GetPrivateIPsByNodeName(name string) ([]string, error)
|
GetPrivateIPsByNodeName(name string) ([]string, error)
|
||||||
|
|
||||||
// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
|
// GetNodeNameByIPConfigurationID gets the nodeName and vmSetName by IP configuration ID.
|
||||||
GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error)
|
GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, string, error)
|
||||||
}
|
}
|
||||||
|
@ -742,8 +742,10 @@ func (ss *scaleSet) getAgentPoolScaleSets(nodes []*v1.Node) (*[]string, error) {
|
|||||||
// for loadbalancer exists then return the eligible VMSet.
|
// for loadbalancer exists then return the eligible VMSet.
|
||||||
func (ss *scaleSet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) (vmSetNames *[]string, err error) {
|
func (ss *scaleSet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) (vmSetNames *[]string, err error) {
|
||||||
hasMode, isAuto, serviceVMSetNames := getServiceLoadBalancerMode(service)
|
hasMode, isAuto, serviceVMSetNames := getServiceLoadBalancerMode(service)
|
||||||
if !hasMode {
|
useSingleSLB := ss.useStandardLoadBalancer() && !ss.EnableMultipleStandardLoadBalancers
|
||||||
// no mode specified in service annotation default to PrimaryScaleSetName.
|
if !hasMode || useSingleSLB {
|
||||||
|
// no mode specified in service annotation or use single SLB mode
|
||||||
|
// default to PrimaryScaleSetName
|
||||||
scaleSetNames := &[]string{ss.Config.PrimaryScaleSetName}
|
scaleSetNames := &[]string{ss.Config.PrimaryScaleSetName}
|
||||||
return scaleSetNames, nil
|
return scaleSetNames, nil
|
||||||
}
|
}
|
||||||
@ -938,9 +940,18 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam
|
|||||||
|
|
||||||
// Check scale set name:
|
// Check scale set name:
|
||||||
// - For basic SKU load balancer, return nil if the node's scale set is mismatched with vmSetName.
|
// - For basic SKU load balancer, return nil if the node's scale set is mismatched with vmSetName.
|
||||||
// - For standard SKU load balancer, backend could belong to multiple VMSS, so we
|
// - For single standard SKU load balancer, backend could belong to multiple VMSS, so we
|
||||||
// don't check vmSet for it.
|
// don't check vmSet for it.
|
||||||
if vmSetName != "" && !ss.useStandardLoadBalancer() && !strings.EqualFold(vmSetName, ssName) {
|
// - For multiple standard SKU load balancers, the behavior is similar to the basic load balancer
|
||||||
|
needCheck := false
|
||||||
|
if !ss.useStandardLoadBalancer() {
|
||||||
|
// need to check the vmSet name when using the basic LB
|
||||||
|
needCheck = true
|
||||||
|
} else if ss.EnableMultipleStandardLoadBalancers {
|
||||||
|
// need to check the vmSet name when using multiple standard LBs
|
||||||
|
needCheck = true
|
||||||
|
}
|
||||||
|
if vmSetName != "" && needCheck && !strings.EqualFold(vmSetName, ssName) {
|
||||||
klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the scaleSet %s", vmName, vmSetName)
|
klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the scaleSet %s", vmName, vmSetName)
|
||||||
return "", "", "", nil, nil
|
return "", "", "", nil, nil
|
||||||
}
|
}
|
||||||
@ -1053,8 +1064,9 @@ func (ss *scaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back
|
|||||||
klog.V(2).Infof("ensureVMSSInPool: ensuring VMSS with backendPoolID %s", backendPoolID)
|
klog.V(2).Infof("ensureVMSSInPool: ensuring VMSS with backendPoolID %s", backendPoolID)
|
||||||
vmssNamesMap := make(map[string]bool)
|
vmssNamesMap := make(map[string]bool)
|
||||||
|
|
||||||
// the standard load balancer supports multiple vmss in its backend while the basic sku doesn't
|
// the single standard load balancer supports multiple vmss in its backend while
|
||||||
if ss.useStandardLoadBalancer() {
|
// multiple standard load balancers and the basic load balancer doesn't
|
||||||
|
if ss.useStandardLoadBalancer() && !ss.EnableMultipleStandardLoadBalancers {
|
||||||
for _, node := range nodes {
|
for _, node := range nodes {
|
||||||
if ss.excludeMasterNodesFromStandardLB() && isMasterNode(node) {
|
if ss.excludeMasterNodesFromStandardLB() && isMasterNode(node) {
|
||||||
continue
|
continue
|
||||||
@ -1363,12 +1375,12 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromNode(nodeName, backendPoolID str
|
|||||||
return nodeResourceGroup, ssName, instanceID, newVM, nil
|
return nodeResourceGroup, ssName, instanceID, newVM, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
|
// GetNodeNameByIPConfigurationID gets the node name and the VMSS name by IP configuration ID.
|
||||||
func (ss *scaleSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
|
func (ss *scaleSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, string, error) {
|
||||||
matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
|
matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
|
||||||
if len(matches) != 4 {
|
if len(matches) != 4 {
|
||||||
klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is mananaged by availability set", ipConfigurationID)
|
klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is managed by availability set", ipConfigurationID)
|
||||||
return "", ErrorNotVmssInstance
|
return "", "", ErrorNotVmssInstance
|
||||||
}
|
}
|
||||||
|
|
||||||
resourceGroup := matches[1]
|
resourceGroup := matches[1]
|
||||||
@ -1376,20 +1388,20 @@ func (ss *scaleSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (st
|
|||||||
instanceID := matches[3]
|
instanceID := matches[3]
|
||||||
vm, err := ss.getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID, azcache.CacheReadTypeUnsafe)
|
vm, err := ss.getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID, azcache.CacheReadTypeUnsafe)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
if vm.OsProfile != nil && vm.OsProfile.ComputerName != nil {
|
if vm.OsProfile != nil && vm.OsProfile.ComputerName != nil {
|
||||||
return strings.ToLower(*vm.OsProfile.ComputerName), nil
|
return strings.ToLower(*vm.OsProfile.ComputerName), scaleSetName, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return "", nil
|
return "", "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getScaleSetAndResourceGroupNameByIPConfigurationID(ipConfigurationID string) (string, string, error) {
|
func getScaleSetAndResourceGroupNameByIPConfigurationID(ipConfigurationID string) (string, string, error) {
|
||||||
matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
|
matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
|
||||||
if len(matches) != 4 {
|
if len(matches) != 4 {
|
||||||
klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is mananaged by availability set", ipConfigurationID)
|
klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is managed by availability set", ipConfigurationID)
|
||||||
return "", "", ErrorNotVmssInstance
|
return "", "", ErrorNotVmssInstance
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1529,7 +1541,7 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nodeName, err := ss.GetNodeNameByIPConfigurationID(ipConfigurationID)
|
nodeName, _, err := ss.GetNodeNameByIPConfigurationID(ipConfigurationID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes.
|
if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes.
|
||||||
continue
|
continue
|
||||||
|
@ -573,19 +573,21 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
|
|||||||
ipConfigurationIDTemplate := "/subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%s/networkInterfaces/%s/ipConfigurations/ipconfig1"
|
ipConfigurationIDTemplate := "/subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%s/networkInterfaces/%s/ipConfigurations/ipconfig1"
|
||||||
|
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
description string
|
description string
|
||||||
scaleSet string
|
scaleSet string
|
||||||
vmList []string
|
vmList []string
|
||||||
ipConfigurationID string
|
ipConfigurationID string
|
||||||
expected string
|
expectedNodeName string
|
||||||
expectError bool
|
expectedScaleSetName string
|
||||||
|
expectError bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
description: "GetNodeNameByIPConfigurationID should get node's Name when the node is existing",
|
description: "GetNodeNameByIPConfigurationID should get node's Name when the node is existing",
|
||||||
scaleSet: "scaleset1",
|
scaleSet: "scaleset1",
|
||||||
ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset1", "0", "scaleset1"),
|
ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset1", "0", "scaleset1"),
|
||||||
vmList: []string{"vmssee6c2000000", "vmssee6c2000001"},
|
vmList: []string{"vmssee6c2000000", "vmssee6c2000001"},
|
||||||
expected: "vmssee6c2000000",
|
expectedNodeName: "vmssee6c2000000",
|
||||||
|
expectedScaleSetName: "scaleset1",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
description: "GetNodeNameByIPConfigurationID should return error for non-exist nodes",
|
description: "GetNodeNameByIPConfigurationID should return error for non-exist nodes",
|
||||||
@ -618,14 +620,15 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
|
|||||||
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
|
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
|
||||||
mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes()
|
mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes()
|
||||||
|
|
||||||
nodeName, err := ss.GetNodeNameByIPConfigurationID(test.ipConfigurationID)
|
nodeName, scalesetName, err := ss.GetNodeNameByIPConfigurationID(test.ipConfigurationID)
|
||||||
if test.expectError {
|
if test.expectError {
|
||||||
assert.Error(t, err, test.description)
|
assert.Error(t, err, test.description)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.NoError(t, err, test.description)
|
assert.NoError(t, err, test.description)
|
||||||
assert.Equal(t, test.expected, nodeName, test.description)
|
assert.Equal(t, test.expectedNodeName, nodeName, test.description)
|
||||||
|
assert.Equal(t, test.expectedScaleSetName, scalesetName, test.description)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1415,6 +1418,7 @@ func TestGetVMSetNames(t *testing.T) {
|
|||||||
description string
|
description string
|
||||||
service *v1.Service
|
service *v1.Service
|
||||||
nodes []*v1.Node
|
nodes []*v1.Node
|
||||||
|
useSingleSLB bool
|
||||||
expectedVMSetNames *[]string
|
expectedVMSetNames *[]string
|
||||||
expectedErr error
|
expectedErr error
|
||||||
}{
|
}{
|
||||||
@ -1423,6 +1427,14 @@ func TestGetVMSetNames(t *testing.T) {
|
|||||||
service: &v1.Service{},
|
service: &v1.Service{},
|
||||||
expectedVMSetNames: &[]string{"vmss"},
|
expectedVMSetNames: &[]string{"vmss"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
description: "GetVMSetNames should return the primary vm set name when using the single SLB",
|
||||||
|
service: &v1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ServiceAnnotationLoadBalancerMode: ServiceAnnotationLoadBalancerAutoModeValue}},
|
||||||
|
},
|
||||||
|
useSingleSLB: true,
|
||||||
|
expectedVMSetNames: &[]string{"vmss"},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
description: "GetVMSetNames should return nil if the service has auto mode annotation",
|
description: "GetVMSetNames should return nil if the service has auto mode annotation",
|
||||||
service: &v1.Service{
|
service: &v1.Service{
|
||||||
@ -1470,6 +1482,11 @@ func TestGetVMSetNames(t *testing.T) {
|
|||||||
ss, err := newTestScaleSet(ctrl)
|
ss, err := newTestScaleSet(ctrl)
|
||||||
assert.NoError(t, err, "unexpected error when creating test VMSS")
|
assert.NoError(t, err, "unexpected error when creating test VMSS")
|
||||||
|
|
||||||
|
if test.useSingleSLB {
|
||||||
|
ss.EnableMultipleStandardLoadBalancers = false
|
||||||
|
ss.LoadBalancerSku = loadBalancerSkuStandard
|
||||||
|
}
|
||||||
|
|
||||||
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
|
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
|
||||||
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
|
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
|
||||||
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
|
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
|
||||||
@ -1663,6 +1680,7 @@ func TestEnsureHostInPool(t *testing.T) {
|
|||||||
vmSetName string
|
vmSetName string
|
||||||
isBasicLB bool
|
isBasicLB bool
|
||||||
isNilVMNetworkConfigs bool
|
isNilVMNetworkConfigs bool
|
||||||
|
useMultipleSLBs bool
|
||||||
expectedNodeResourceGroup string
|
expectedNodeResourceGroup string
|
||||||
expectedVMSSName string
|
expectedVMSSName string
|
||||||
expectedInstanceID string
|
expectedInstanceID string
|
||||||
@ -1675,6 +1693,12 @@ func TestEnsureHostInPool(t *testing.T) {
|
|||||||
vmSetName: "vmss-1",
|
vmSetName: "vmss-1",
|
||||||
isBasicLB: true,
|
isBasicLB: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
description: "EnsureHostInPool should skip the current node if the vmSetName is not equal to the node's vmss name and multiple SLBs are used",
|
||||||
|
nodeName: "vmss-vm-000000",
|
||||||
|
vmSetName: "vmss-1",
|
||||||
|
useMultipleSLBs: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
description: "EnsureHostInPool should skip the current node if the network configs of the VMSS VM is nil",
|
description: "EnsureHostInPool should skip the current node if the network configs of the VMSS VM is nil",
|
||||||
nodeName: "vmss-vm-000000",
|
nodeName: "vmss-vm-000000",
|
||||||
@ -1749,6 +1773,10 @@ func TestEnsureHostInPool(t *testing.T) {
|
|||||||
ss.LoadBalancerSku = loadBalancerSkuStandard
|
ss.LoadBalancerSku = loadBalancerSkuStandard
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if test.useMultipleSLBs {
|
||||||
|
ss.EnableMultipleStandardLoadBalancers = true
|
||||||
|
}
|
||||||
|
|
||||||
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
|
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
|
||||||
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
|
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
|
||||||
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
|
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
|
||||||
@ -1798,6 +1826,7 @@ func TestEnsureVMSSInPool(t *testing.T) {
|
|||||||
expectedPutVMSS bool
|
expectedPutVMSS bool
|
||||||
setIPv6Config bool
|
setIPv6Config bool
|
||||||
clusterIP string
|
clusterIP string
|
||||||
|
useMultipleSLBs bool
|
||||||
expectedErr error
|
expectedErr error
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@ -1907,6 +1936,34 @@ func TestEnsureVMSSInPool(t *testing.T) {
|
|||||||
clusterIP: "fd00::e68b",
|
clusterIP: "fd00::e68b",
|
||||||
expectedPutVMSS: true,
|
expectedPutVMSS: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
description: "ensureVMSSInPool should work for the basic load balancer",
|
||||||
|
nodes: []*v1.Node{
|
||||||
|
{
|
||||||
|
Spec: v1.NodeSpec{
|
||||||
|
ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
vmSetName: testVMSSName,
|
||||||
|
isBasicLB: true,
|
||||||
|
backendPoolID: testLBBackendpoolID1,
|
||||||
|
expectedPutVMSS: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "ensureVMSSInPool should work for multiple standard load balancers",
|
||||||
|
nodes: []*v1.Node{
|
||||||
|
{
|
||||||
|
Spec: v1.NodeSpec{
|
||||||
|
ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
vmSetName: testVMSSName,
|
||||||
|
backendPoolID: testLBBackendpoolID1,
|
||||||
|
useMultipleSLBs: true,
|
||||||
|
expectedPutVMSS: true,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range testCases {
|
for _, test := range testCases {
|
||||||
|
@ -293,12 +293,13 @@ func (mr *MockVMSetMockRecorder) GetPrivateIPsByNodeName(name interface{}) *gomo
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetNodeNameByIPConfigurationID mocks base method
|
// GetNodeNameByIPConfigurationID mocks base method
|
||||||
func (m *MockVMSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
|
func (m *MockVMSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, string, error) {
|
||||||
m.ctrl.T.Helper()
|
m.ctrl.T.Helper()
|
||||||
ret := m.ctrl.Call(m, "GetNodeNameByIPConfigurationID", ipConfigurationID)
|
ret := m.ctrl.Call(m, "GetNodeNameByIPConfigurationID", ipConfigurationID)
|
||||||
ret0, _ := ret[0].(string)
|
ret0, _ := ret[0].(string)
|
||||||
ret1, _ := ret[1].(error)
|
ret1, _ := ret[1].(string)
|
||||||
return ret0, ret1
|
ret2, _ := ret[2].(error)
|
||||||
|
return ret0, ret1, ret2
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetNodeNameByIPConfigurationID indicates an expected call of GetNodeNameByIPConfigurationID
|
// GetNodeNameByIPConfigurationID indicates an expected call of GetNodeNameByIPConfigurationID
|
||||||
|
Loading…
Reference in New Issue
Block a user