diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common_test.go index 2599b9451d3..44783eb8420 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common_test.go @@ -211,7 +211,7 @@ func TestCommonAttachDiskWithVMSS(t *testing.T) { mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(testCloud, testVMSSName, "", 0, test.vmssList, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(testCloud, testVMSSName, "", 0, test.vmssList, "", false) mockVMSSVMClient := testCloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go index 18afa6046f2..3324929d460 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go @@ -98,13 +98,13 @@ func TestAttachDiskWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}) + expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any()).Return(nil).MaxTimes(1) - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(testCloud, scaleSetName, "", 0, test.vmssVMList, "succeeded") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(testCloud, scaleSetName, "", 0, test.vmssVMList, "succeeded", false) for _, vmssvm := range expectedVMSSVMs { vmssvm.StorageProfile = &compute.StorageProfile{ OsDisk: &compute.OSDisk{ @@ -194,13 +194,13 @@ func TestDetachDiskWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}) + expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any()).Return(nil).MaxTimes(1) - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(testCloud, scaleSetName, "", 0, test.vmssVMList, "succeeded") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(testCloud, scaleSetName, "", 0, test.vmssVMList, "succeeded", false) for _, vmssvm := range expectedVMSSVMs { vmssvm.StorageProfile = &compute.StorageProfile{ OsDisk: &compute.OSDisk{ @@ -297,13 +297,13 @@ func TestGetDataDisksWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}) + expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any()).Return(nil).MaxTimes(1) - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(testCloud, scaleSetName, "", 0, []string{"vmss00-vm-000000"}, "succeeded") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(testCloud, scaleSetName, "", 0, []string{"vmss00-vm-000000"}, "succeeded", false) if !test.isDataDiskNull { for _, vmssvm := range expectedVMSSVMs { vmssvm.StorageProfile = &compute.StorageProfile{ diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go index 9831bf0e7da..7670608ca14 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go @@ -821,7 +821,7 @@ func (ss *scaleSet) getConfigForScaleSetByIPFamily(config *compute.VirtualMachin } } - return nil, fmt.Errorf("failed to find a IPconfiguration(IPv6=%v) for the scale set VM %q", IPv6, nodeName) + return nil, fmt.Errorf("failed to find a IPconfiguration(IPv6=%v) for the scale set VM %q", IPv6, nodeName) } // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is @@ -865,6 +865,9 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam return "", "", "", nil, err } } else { + // For IPv6 or dualstack service, we need to pick the right IP configuration based on the cluster ip family + // IPv6 configuration is only supported as non-primary, so we need to fetch the ip configuration where the + // privateIPAddressVersion matches the clusterIP family primaryIPConfiguration, err = ss.getConfigForScaleSetByIPFamily(primaryNetworkInterfaceConfiguration, vmName, ipv6) if err != nil { return "", "", "", nil, err @@ -998,10 +1001,22 @@ func (ss *scaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back if err != nil { return err } - primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC) - if err != nil { - return err + var primaryIPConfig *compute.VirtualMachineScaleSetIPConfiguration + ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + // Find primary network interface configuration. + if !ss.Cloud.ipv6DualStackEnabled && !ipv6 { + // Find primary IP configuration. + primaryIPConfig, err = getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC) + if err != nil { + return err + } + } else { + primaryIPConfig, err = ss.getConfigForScaleSetByIPFamily(primaryNIC, "", ipv6) + if err != nil { + return err + } } + loadBalancerBackendAddressPools := []compute.SubResource{} if primaryIPConfig.LoadBalancerBackendAddressPools != nil { loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go index d83aa6aa127..07fb2073921 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go @@ -99,7 +99,7 @@ func TestVMSSVMCache(t *testing.T) { } mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() - expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, vmssName, "", 0, vmList, "") + expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, vmssName, "", 0, vmList, "", false) mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() // validate getting VMSS VM via cache. @@ -154,7 +154,7 @@ func TestVMSSVMCacheWithDeletingNodes(t *testing.T) { } mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() - expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, vmssName, "", 0, vmList, string(compute.ProvisioningStateDeleting)) + expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, vmssName, "", 0, vmList, string(compute.ProvisioningStateDeleting), false) mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() for i := range expectedVMs { diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go index 2f328a01640..c0eb15fe87f 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go @@ -67,11 +67,26 @@ func newTestScaleSetWithState(ctrl *gomock.Controller) (*scaleSet, error) { return ss.(*scaleSet), nil } -func buildTestVMSS(name string, lbBackendpoolIDs []string) compute.VirtualMachineScaleSet { +func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.VirtualMachineScaleSet { lbBackendpools := make([]compute.SubResource, 0) for _, id := range lbBackendpoolIDs { lbBackendpools = append(lbBackendpools, compute.SubResource{ID: to.StringPtr(id)}) } + ipConfig := []compute.VirtualMachineScaleSetIPConfiguration{ + { + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + LoadBalancerBackendAddressPools: &lbBackendpools, + }, + }, + } + if ipv6 { + ipConfig = append(ipConfig, compute.VirtualMachineScaleSetIPConfiguration{ + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + LoadBalancerBackendAddressPools: &lbBackendpools, + PrivateIPAddressVersion: compute.IPv6, + }, + }) + } expectedVMSS := compute.VirtualMachineScaleSet{ Name: &name, @@ -82,14 +97,8 @@ func buildTestVMSS(name string, lbBackendpoolIDs []string) compute.VirtualMachin NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ { VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - Primary: to.BoolPtr(true), - IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ - { - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - LoadBalancerBackendAddressPools: &lbBackendpools, - }, - }, - }, + Primary: to.BoolPtr(true), + IPConfigurations: &ipConfig, }, }, }, @@ -101,7 +110,7 @@ func buildTestVMSS(name string, lbBackendpoolIDs []string) compute.VirtualMachin return expectedVMSS } -func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomain int32, vmList []string, state string) ([]compute.VirtualMachineScaleSetVM, network.Interface, network.PublicIPAddress) { +func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomain int32, vmList []string, state string, isIPv6 bool) ([]compute.VirtualMachineScaleSetVM, network.Interface, network.PublicIPAddress) { expectedVMSSVMs := make([]compute.VirtualMachineScaleSetVM, 0) expectedInterface := network.Interface{} expectedPIP := network.PublicIPAddress{} @@ -141,6 +150,25 @@ func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomai }, }, } + if isIPv6 { + networkConfigurations = append(networkConfigurations, compute.VirtualMachineScaleSetNetworkConfiguration{ + Name: to.StringPtr("ipconfig1v6"), + ID: to.StringPtr("fakeNetworkConfigurationIPv6"), + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + Name: to.StringPtr("ipconfig1"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: to.BoolPtr(false), + LoadBalancerBackendAddressPools: &[]compute.SubResource{{ID: to.StringPtr(testLBBackendpoolID0)}}, + PrivateIPAddressVersion: compute.IPv6, + }, + }, + }, + }, + }) + } + vmssVM := compute.VirtualMachineScaleSetVM{ VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ ProvisioningState: to.StringPtr(state), @@ -285,7 +313,7 @@ func TestGetInstanceIDByNodeName(t *testing.T) { } mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() - expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "") + expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() mockVMsClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) @@ -375,7 +403,7 @@ func TestGetZoneByNodeName(t *testing.T) { } mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() - expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, test.zone, test.faultDomain, test.vmList, "") + expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, test.zone, test.faultDomain, test.vmList, "", false) mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() mockVMsClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) @@ -442,7 +470,7 @@ func TestGetIPByNodeName(t *testing.T) { } mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() - expectedVMs, expectedInterface, expectedPIP := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "") + expectedVMs, expectedInterface, expectedPIP := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() mockInterfaceClient.EXPECT().GetVirtualMachineScaleSetNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedInterface, nil).AnyTimes() mockPIPClient.EXPECT().GetVirtualMachineScaleSetPublicIPAddress(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedPIP, nil).AnyTimes() @@ -515,7 +543,7 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) { } mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() - expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "") + expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() nodeName, err := ss.getNodeNameByIPConfigurationID(test.ipConfigurationID) @@ -660,7 +688,7 @@ func TestGetVmssVM(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.existedVMSSName, "", 0, test.existedNodeNames, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.existedVMSSName, "", 0, test.existedNodeNames, "", false) var expectedVMSSVM compute.VirtualMachineScaleSetVM for _, expected := range expectedVMSSVMs { if strings.EqualFold(*expected.OsProfile.ComputerName, test.nodeName) { @@ -716,7 +744,7 @@ func TestGetPowerStatusByNodeName(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "", false) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) if test.nilStatus { expectedVMSSVMs[0].InstanceView.Statuses = nil @@ -762,7 +790,7 @@ func TestGetVmssVMByInstanceID(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "", false) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() @@ -810,7 +838,7 @@ func TestGetInstanceTypeByNodeName(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "", false) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() @@ -967,7 +995,7 @@ func TestGetPrimaryInterface(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, test.vmssClientErr).AnyTimes() - expectedVMSSVMs, expectedInterface, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + expectedVMSSVMs, expectedInterface, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "", false) if !test.hasPrimaryInterface { networkInterfaces := *expectedVMSSVMs[0].NetworkProfile.NetworkInterfaces networkInterfaces[0].Primary = to.BoolPtr(false) @@ -1095,7 +1123,7 @@ func TestGetPrivateIPsByNodeName(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() - expectedVMSSVMs, expectedInterface, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + expectedVMSSVMs, expectedInterface, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "", false) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() @@ -1684,7 +1712,7 @@ func TestEnsureHostInPool(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{string(test.nodeName)}, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{string(test.nodeName)}, "", false) if test.isNilVMNetworkConfigs { expectedVMSSVMs[0].NetworkProfileConfiguration.NetworkInterfaceConfigurations = nil } @@ -1727,6 +1755,8 @@ func TestEnsureVMSSInPool(t *testing.T) { isVMSSDeallocating bool isVMSSNilNICConfig bool expectedPutVMSS bool + setIPv6Config bool + clusterIP string expectedErr error }{ { @@ -1806,6 +1836,36 @@ func TestEnsureVMSSInPool(t *testing.T) { backendPoolID: testLBBackendpoolID1, expectedPutVMSS: true, }, + { + description: "ensureVMSSInPool should fail if no IPv6 network config", + nodes: []*v1.Node{ + { + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0", + }, + }, + }, + isBasicLB: false, + backendPoolID: testLBBackendpoolID1, + clusterIP: "fd00::e68b", + expectedPutVMSS: false, + expectedErr: fmt.Errorf("failed to find a IPconfiguration(IPv6=true) for the scale set VM \"\""), + }, + { + description: "ensureVMSSInPool should update the VMSS correctly for IPv6", + nodes: []*v1.Node{ + { + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0", + }, + }, + }, + isBasicLB: false, + backendPoolID: testLBBackendpoolID1, + setIPv6Config: true, + clusterIP: "fd00::e68b", + expectedPutVMSS: true, + }, } for _, test := range testCases { @@ -1816,7 +1876,7 @@ func TestEnsureVMSSInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard } - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}) + expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, test.setIPv6Config) if test.isVMSSDeallocating { expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating } @@ -1832,11 +1892,11 @@ func TestEnsureVMSSInPool(t *testing.T) { } mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(nil).Times(vmssPutTimes) - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "", test.setIPv6Config) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() - err = ss.ensureVMSSInPool(&v1.Service{}, test.nodes, test.backendPoolID, test.vmSetName) + err = ss.ensureVMSSInPool(&v1.Service{Spec: v1.ServiceSpec{ClusterIP: test.clusterIP}}, test.nodes, test.backendPoolID, test.vmSetName) assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") } } @@ -1913,13 +1973,13 @@ func TestEnsureHostsInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard ss.ExcludeMasterFromStandardLB = to.BoolPtr(true) - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}) + expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(nil).MaxTimes(1) - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "", false) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(nil).Times(test.expectedVMSSVMPutTimes) @@ -2010,7 +2070,7 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "", false) if test.isNilVMNetworkConfigs { expectedVMSSVMs[0].NetworkProfileConfiguration.NetworkInterfaceConfigurations = nil } @@ -2095,7 +2155,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}) + expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) if test.isVMSSDeallocating { expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating } @@ -2111,7 +2171,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) { } mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(test.vmssClientErr).Times(vmssPutTimes) - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "", false) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() @@ -2189,13 +2249,13 @@ func TestEnsureBackendPoolDeleted(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.description) - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}) + expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(nil).MaxTimes(1) - expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "") + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "", false) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(test.vmClientErr).Times(test.expectedVMSSVMPutTimes) @@ -2248,11 +2308,11 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) { }, }, } - vmss0 := buildTestVMSS("vmss-0", []string{testLBBackendpoolID0, testLBBackendpoolID1}) - vmss1 := buildTestVMSS("vmss-1", []string{testLBBackendpoolID0, testLBBackendpoolID1}) + vmss0 := buildTestVMSS("vmss-0", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) + vmss1 := buildTestVMSS("vmss-1", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) - expectedVMSSVMsOfVMSS0, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-0", "", 0, []string{"vmss-vm-000000"}, "succeeded") - expectedVMSSVMsOfVMSS1, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-1", "", 0, []string{"vmss-vm-000001"}, "succeeded") + expectedVMSSVMsOfVMSS0, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-0", "", 0, []string{"vmss-vm-000000"}, "succeeded", false) + expectedVMSSVMsOfVMSS1, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-1", "", 0, []string{"vmss-vm-000001"}, "succeeded", false) for _, expectedVMSSVMs := range [][]compute.VirtualMachineScaleSetVM{expectedVMSSVMsOfVMSS0, expectedVMSSVMsOfVMSS1} { vmssVMNetworkConfigs := expectedVMSSVMs[0].NetworkProfileConfiguration vmssVMIPConfigs := (*vmssVMNetworkConfigs.NetworkInterfaceConfigurations)[0].VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations