Merge pull request #80485 from khenidak/lb-dualstack-support

Azure support for dualstack LB services

staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go

@@ -35,6 +35,9 @@ import (
 	cloudprovider "k8s.io/cloud-provider"
 	servicehelpers "k8s.io/cloud-provider/service/helpers"
 	"k8s.io/klog"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilnet "k8s.io/utils/net"
 )
 
 const (
@@ -536,6 +539,23 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai
 		}
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) {
+		// TODO: (khenidak) if we ever enable IPv6 single stack, then we should
+		// not wrap the following in a feature gate
+		ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP)
+		if ipv6 {
+			pip.PublicIPAddressVersion = network.IPv6
+			klog.V(2).Infof("service(%s): pip(%s) - creating as ipv6 for clusterIP:%v", serviceName, *pip.Name, service.Spec.ClusterIP)
+			// static allocation on IPv6 on Azure is not allowed
+			pip.PublicIPAddressPropertiesFormat.PublicIPAllocationMethod = network.Dynamic
+		} else {
+			pip.PublicIPAddressVersion = network.IPv4
+			klog.V(2).Infof("service(%s): pip(%s) - creating as ipv4 for clusterIP:%v", serviceName, *pip.Name, service.Spec.ClusterIP)
+		}
+	}
+
+	klog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - creating", serviceName, *pip.Name)
+
 	klog.V(10).Infof("CreateOrUpdatePIP(%s, %q): start", pipResourceGroup, *pip.Name)
 	err = az.CreateOrUpdatePIP(service, pipResourceGroup, pip)
 	if err != nil {
@@ -649,7 +669,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 	klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) wantLb(%t) resolved load balancer name", serviceName, lbName, wantLb)
 	lbFrontendIPConfigName := az.getFrontendIPConfigName(service, subnet(service))
 	lbFrontendIPConfigID := az.getFrontendIPConfigID(lbName, lbFrontendIPConfigName)
-	lbBackendPoolName := getBackendPoolName(clusterName)
+	lbBackendPoolName := getBackendPoolName(clusterName, service)
 	lbBackendPoolID := az.getBackendPoolID(lbName, lbBackendPoolName)
 
 	lbIdleTimeout, err := getIdleTimeout(service)
@@ -727,6 +747,13 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 			// construct FrontendIPConfigurationPropertiesFormat
 			var fipConfigurationProperties *network.FrontendIPConfigurationPropertiesFormat
 			if isInternal {
+				// azure does not support ILB for IPv6 yet.
+				// TODO: remove this check when ILB supports IPv6 *and* the SDK
+				// have been rev'ed to 2019* version
+				if utilnet.IsIPv6String(service.Spec.ClusterIP) {
+					return nil, fmt.Errorf("ensure(%s): lb(%s) - internal load balancers does not support IPv6", serviceName, lbName)
+				}
+
 				subnetName := subnet(service)
 				if subnetName == nil {
 					subnetName = &az.SubnetName
@@ -1025,11 +1052,18 @@ func (az *Cloud) reconcileLoadBalancerRule(
 					LoadDistribution:    loadDistribution,
 					FrontendPort:        to.Int32Ptr(port.Port),
 					BackendPort:         to.Int32Ptr(port.Port),
-					EnableFloatingIP:    to.BoolPtr(true),
 					DisableOutboundSnat: to.BoolPtr(az.disableLoadBalancerOutboundSNAT()),
 					EnableTCPReset:      enableTCPReset,
 				},
 			}
+			// LB does not support floating IPs for IPV6 rules
+			if utilnet.IsIPv6String(service.Spec.ClusterIP) {
+				expectedRule.BackendPort = to.Int32Ptr(port.NodePort)
+				expectedRule.EnableFloatingIP = to.BoolPtr(false)
+			} else {
+				expectedRule.EnableFloatingIP = to.BoolPtr(true)
+			}
+
 			if protocol == v1.ProtocolTCP {
 				expectedRule.LoadBalancingRulePropertiesFormat.IdleTimeoutInMinutes = lbIdleTimeout
 			}

staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go

@@ -34,15 +34,13 @@ import (
 	// Azure route controller changes behavior if ipv6dual stack feature is turned on
 	// remove this once the feature graduates
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	"k8s.io/component-base/featuregate"
 )
 
 // copied to minimize the number of cross reference
 // and exceptions in publishing and allowed imports.
 const (
-	IPv6DualStack      featuregate.Feature = "IPv6DualStack"
-	routeNameFmt                           = "%s____%s"
-	routeNameSeparator                     = "____"
+	routeNameFmt       = "%s____%s"
+	routeNameSeparator = "____"
 )
 
 // ListRoutes lists all managed routes that belong to the specified clusterName

staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go

@@ -39,9 +39,17 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/klog"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/component-base/featuregate"
+	utilnet "k8s.io/utils/net"
 )
 
 const (
+	// IPv6DualStack is here to avoid having to import features pkg
+	// and violate import rules
+	IPv6DualStack featuregate.Feature = "IPv6DualStack"
+
 	loadBalancerMinimumPriority = 500
 	loadBalancerMaximumPriority = 4096
 
@@ -220,11 +228,50 @@ func getPrimaryIPConfig(nic network.Interface) (*network.InterfaceIPConfiguratio
 	return nil, fmt.Errorf("failed to determine the primary ipconfig. nicname=%q", *nic.Name)
 }
 
+// returns first ip configuration on a nic by family
+func getIPConfigByIPFamily(nic network.Interface, IPv6 bool) (*network.InterfaceIPConfiguration, error) {
+	if nic.IPConfigurations == nil {
+		return nil, fmt.Errorf("nic.IPConfigurations for nic (nicname=%q) is nil", *nic.Name)
+	}
+
+	var ipVersion network.IPVersion
+	if IPv6 {
+		ipVersion = network.IPv6
+	} else {
+		ipVersion = network.IPv4
+	}
+	for _, ref := range *nic.IPConfigurations {
+		if ref.PrivateIPAddress != nil && ref.PrivateIPAddressVersion == ipVersion {
+			return &ref, nil
+		}
+	}
+	return nil, fmt.Errorf("failed to determine the ipconfig(IPv6=%v). nicname=%q", IPv6, *nic.Name)
+}
+
 func isInternalLoadBalancer(lb *network.LoadBalancer) bool {
 	return strings.HasSuffix(*lb.Name, InternalLoadBalancerNameSuffix)
 }
 
-func getBackendPoolName(clusterName string) string {
+// getBackendPoolName the LB BackendPool name for a service.
+// to ensure backword and forward compat:
+// SingleStack -v4 (pre v1.16) => BackendPool name == clusterName
+// SingleStack -v6 => BackendPool name == clusterName (all cluster bootstrap uses this name)
+// DualStack
+//	=> IPv4 BackendPool name == clusterName
+//  => IPv6 BackendPool name == <clusterName>-IPv6
+// This means:
+// clusters moving from IPv4 to duakstack will require no changes
+// clusters moving from IPv6 (while not seen in the wild, we can not rule out their existence)
+// to dualstack will require deleting backend pools (the reconciler will take care of creating correct backendpools)
+func getBackendPoolName(clusterName string, service *v1.Service) string {
+	if !utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) {
+		return clusterName
+	}
+	IPv6 := utilnet.IsIPv6String(service.Spec.ClusterIP)
+	if IPv6 {
+		return fmt.Sprintf("%v-IPv6", clusterName)
+	}
+
 	return clusterName
 }
 
@@ -674,9 +721,17 @@ func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types.
 	}
 
 	var primaryIPConfig *network.InterfaceIPConfiguration
-	primaryIPConfig, err = getPrimaryIPConfig(nic)
-	if err != nil {
-		return err
+	if !utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) {
+		primaryIPConfig, err = getPrimaryIPConfig(nic)
+		if err != nil {
+			return err
+		}
+	} else {
+		ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP)
+		primaryIPConfig, err = getIPConfigByIPFamily(nic, ipv6)
+		if err != nil {
+			return err
+		}
 	}
 
 	foundPool := false

staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go

@@ -35,6 +35,9 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/klog"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilnet "k8s.io/utils/net"
 )
 
 var (
@@ -684,6 +687,25 @@ func getPrimaryIPConfigFromVMSSNetworkConfig(config *compute.VirtualMachineScale
 	return nil, fmt.Errorf("failed to find a primary IP configuration")
 }
 
+func (ss *scaleSet) getConfigForScaleSetByIPFamily(config *compute.VirtualMachineScaleSetNetworkConfiguration, nodeName string, IPv6 bool) (*compute.VirtualMachineScaleSetIPConfiguration, error) {
+	ipConfigurations := *config.IPConfigurations
+
+	var ipVersion compute.IPVersion
+	if IPv6 {
+		ipVersion = compute.IPv6
+	} else {
+		ipVersion = compute.IPv4
+	}
+	for idx := range ipConfigurations {
+		ipConfig := &ipConfigurations[idx]
+		if ipConfig.PrivateIPAddressVersion == ipVersion {
+			return ipConfig, nil
+		}
+	}
+
+	return nil, fmt.Errorf("failed to find a  IPconfiguration(IPv6=%v) for the scale set VM %q", IPv6, nodeName)
+}
+
 // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
 // participating in the specified LoadBalancer Backend Pool.
 func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) error {
@@ -708,16 +730,27 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam
 		klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vm %s, probably because the vm's being deleted", vmName)
 		return nil
 	}
+
 	networkInterfaceConfigurations := *vm.NetworkProfileConfiguration.NetworkInterfaceConfigurations
 	primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, vmName)
 	if err != nil {
 		return err
 	}
 
-	// Find primary IP configuration.
-	primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration)
-	if err != nil {
-		return err
+	var primaryIPConfiguration *compute.VirtualMachineScaleSetIPConfiguration
+	// Find primary network interface configuration.
+	if !utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) {
+		// Find primary IP configuration.
+		primaryIPConfiguration, err = getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration)
+		if err != nil {
+			return err
+		}
+	} else {
+		ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP)
+		primaryIPConfiguration, err = ss.getConfigForScaleSetByIPFamily(primaryNetworkInterfaceConfiguration, vmName, ipv6)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Update primary IP configuration's LoadBalancerBackendAddressPools.