From bd9108ba8454a390942a6ec7b8eefdc028fecc52 Mon Sep 17 00:00:00 2001 From: "Khaled Henidak(Kal)" Date: Thu, 22 Aug 2019 17:39:25 +0000 Subject: [PATCH] azure support for ipv6 and dual stack services(excluding ILB) --- .../azure/azure_loadbalancer.go | 38 ++++++++++- .../azure/azure_routes.go | 6 +- .../azure/azure_standard.go | 63 +++++++++++++++++-- .../azure/azure_vmss.go | 41 ++++++++++-- 4 files changed, 134 insertions(+), 14 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go index 19529177ced..8dfeb4f3a80 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go @@ -35,6 +35,9 @@ import ( cloudprovider "k8s.io/cloud-provider" servicehelpers "k8s.io/cloud-provider/service/helpers" "k8s.io/klog" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilnet "k8s.io/utils/net" ) const ( @@ -536,6 +539,23 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai } } + if utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) { + // TODO: (khenidak) if we ever enable IPv6 single stack, then we should + // not wrap the following in a feature gate + ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + if ipv6 { + pip.PublicIPAddressVersion = network.IPv6 + klog.V(2).Infof("service(%s): pip(%s) - creating as ipv6 for clusterIP:%v", serviceName, *pip.Name, service.Spec.ClusterIP) + // static allocation on IPv6 on Azure is not allowed + pip.PublicIPAddressPropertiesFormat.PublicIPAllocationMethod = network.Dynamic + } else { + pip.PublicIPAddressVersion = network.IPv4 + klog.V(2).Infof("service(%s): pip(%s) - creating as ipv4 for clusterIP:%v", serviceName, *pip.Name, service.Spec.ClusterIP) + } + } + + klog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - creating", serviceName, *pip.Name) + klog.V(10).Infof("CreateOrUpdatePIP(%s, %q): start", pipResourceGroup, *pip.Name) err = az.CreateOrUpdatePIP(service, pipResourceGroup, pip) if err != nil { @@ -649,7 +669,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) wantLb(%t) resolved load balancer name", serviceName, lbName, wantLb) lbFrontendIPConfigName := az.getFrontendIPConfigName(service, subnet(service)) lbFrontendIPConfigID := az.getFrontendIPConfigID(lbName, lbFrontendIPConfigName) - lbBackendPoolName := getBackendPoolName(clusterName) + lbBackendPoolName := getBackendPoolName(clusterName, service) lbBackendPoolID := az.getBackendPoolID(lbName, lbBackendPoolName) lbIdleTimeout, err := getIdleTimeout(service) @@ -727,6 +747,13 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, // construct FrontendIPConfigurationPropertiesFormat var fipConfigurationProperties *network.FrontendIPConfigurationPropertiesFormat if isInternal { + // azure does not support ILB for IPv6 yet. + // TODO: remove this check when ILB supports IPv6 *and* the SDK + // have been rev'ed to 2019* version + if utilnet.IsIPv6String(service.Spec.ClusterIP) { + return nil, fmt.Errorf("ensure(%s): lb(%s) - internal load balancers does not support IPv6", serviceName, lbName) + } + subnetName := subnet(service) if subnetName == nil { subnetName = &az.SubnetName @@ -1025,11 +1052,18 @@ func (az *Cloud) reconcileLoadBalancerRule( LoadDistribution: loadDistribution, FrontendPort: to.Int32Ptr(port.Port), BackendPort: to.Int32Ptr(port.Port), - EnableFloatingIP: to.BoolPtr(true), DisableOutboundSnat: to.BoolPtr(az.disableLoadBalancerOutboundSNAT()), EnableTCPReset: enableTCPReset, }, } + // LB does not support floating IPs for IPV6 rules + if utilnet.IsIPv6String(service.Spec.ClusterIP) { + expectedRule.BackendPort = to.Int32Ptr(port.NodePort) + expectedRule.EnableFloatingIP = to.BoolPtr(false) + } else { + expectedRule.EnableFloatingIP = to.BoolPtr(true) + } + if protocol == v1.ProtocolTCP { expectedRule.LoadBalancingRulePropertiesFormat.IdleTimeoutInMinutes = lbIdleTimeout } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go index 028bb445d03..3cc689fa597 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go @@ -34,15 +34,13 @@ import ( // Azure route controller changes behavior if ipv6dual stack feature is turned on // remove this once the feature graduates utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/component-base/featuregate" ) // copied to minimize the number of cross reference // and exceptions in publishing and allowed imports. const ( - IPv6DualStack featuregate.Feature = "IPv6DualStack" - routeNameFmt = "%s____%s" - routeNameSeparator = "____" + routeNameFmt = "%s____%s" + routeNameSeparator = "____" ) // ListRoutes lists all managed routes that belong to the specified clusterName diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go index da58cd43220..444584edf37 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go @@ -39,9 +39,17 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" + utilnet "k8s.io/utils/net" ) const ( + // IPv6DualStack is here to avoid having to import features pkg + // and violate import rules + IPv6DualStack featuregate.Feature = "IPv6DualStack" + loadBalancerMinimumPriority = 500 loadBalancerMaximumPriority = 4096 @@ -220,11 +228,50 @@ func getPrimaryIPConfig(nic network.Interface) (*network.InterfaceIPConfiguratio return nil, fmt.Errorf("failed to determine the primary ipconfig. nicname=%q", *nic.Name) } +// returns first ip configuration on a nic by family +func getIPConfigByIPFamily(nic network.Interface, IPv6 bool) (*network.InterfaceIPConfiguration, error) { + if nic.IPConfigurations == nil { + return nil, fmt.Errorf("nic.IPConfigurations for nic (nicname=%q) is nil", *nic.Name) + } + + var ipVersion network.IPVersion + if IPv6 { + ipVersion = network.IPv6 + } else { + ipVersion = network.IPv4 + } + for _, ref := range *nic.IPConfigurations { + if ref.PrivateIPAddress != nil && ref.PrivateIPAddressVersion == ipVersion { + return &ref, nil + } + } + return nil, fmt.Errorf("failed to determine the ipconfig(IPv6=%v). nicname=%q", IPv6, *nic.Name) +} + func isInternalLoadBalancer(lb *network.LoadBalancer) bool { return strings.HasSuffix(*lb.Name, InternalLoadBalancerNameSuffix) } -func getBackendPoolName(clusterName string) string { +// getBackendPoolName the LB BackendPool name for a service. +// to ensure backword and forward compat: +// SingleStack -v4 (pre v1.16) => BackendPool name == clusterName +// SingleStack -v6 => BackendPool name == clusterName (all cluster bootstrap uses this name) +// DualStack +// => IPv4 BackendPool name == clusterName +// => IPv6 BackendPool name == -IPv6 +// This means: +// clusters moving from IPv4 to duakstack will require no changes +// clusters moving from IPv6 (while not seen in the wild, we can not rule out their existence) +// to dualstack will require deleting backend pools (the reconciler will take care of creating correct backendpools) +func getBackendPoolName(clusterName string, service *v1.Service) string { + if !utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) { + return clusterName + } + IPv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + if IPv6 { + return fmt.Sprintf("%v-IPv6", clusterName) + } + return clusterName } @@ -674,9 +721,17 @@ func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types. } var primaryIPConfig *network.InterfaceIPConfiguration - primaryIPConfig, err = getPrimaryIPConfig(nic) - if err != nil { - return err + if !utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) { + primaryIPConfig, err = getPrimaryIPConfig(nic) + if err != nil { + return err + } + } else { + ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + primaryIPConfig, err = getIPConfigByIPFamily(nic, ipv6) + if err != nil { + return err + } } foundPool := false diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go index 3a39b0c179f..040de1d88b0 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go @@ -35,6 +35,9 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilnet "k8s.io/utils/net" ) var ( @@ -684,6 +687,25 @@ func getPrimaryIPConfigFromVMSSNetworkConfig(config *compute.VirtualMachineScale return nil, fmt.Errorf("failed to find a primary IP configuration") } +func (ss *scaleSet) getConfigForScaleSetByIPFamily(config *compute.VirtualMachineScaleSetNetworkConfiguration, nodeName string, IPv6 bool) (*compute.VirtualMachineScaleSetIPConfiguration, error) { + ipConfigurations := *config.IPConfigurations + + var ipVersion compute.IPVersion + if IPv6 { + ipVersion = compute.IPv6 + } else { + ipVersion = compute.IPv4 + } + for idx := range ipConfigurations { + ipConfig := &ipConfigurations[idx] + if ipConfig.PrivateIPAddressVersion == ipVersion { + return ipConfig, nil + } + } + + return nil, fmt.Errorf("failed to find a IPconfiguration(IPv6=%v) for the scale set VM %q", IPv6, nodeName) +} + // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is // participating in the specified LoadBalancer Backend Pool. func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) error { @@ -708,16 +730,27 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vm %s, probably because the vm's being deleted", vmName) return nil } + networkInterfaceConfigurations := *vm.NetworkProfileConfiguration.NetworkInterfaceConfigurations primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, vmName) if err != nil { return err } - // Find primary IP configuration. - primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration) - if err != nil { - return err + var primaryIPConfiguration *compute.VirtualMachineScaleSetIPConfiguration + // Find primary network interface configuration. + if !utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) { + // Find primary IP configuration. + primaryIPConfiguration, err = getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration) + if err != nil { + return err + } + } else { + ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + primaryIPConfiguration, err = ss.getConfigForScaleSetByIPFamily(primaryNetworkInterfaceConfiguration, vmName, ipv6) + if err != nil { + return err + } } // Update primary IP configuration's LoadBalancerBackendAddressPools.