From 7a9bca38886f5cbcd6e33fc713475dea0d9b8b89 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 24 Sep 2024 15:19:50 +0000 Subject: [PATCH] bugfix: initialize secondary range registry with the right value When MultiCIDRServiceAllocator feature is enabled, we added an additional feature gate DisableAllocatorDualWrite that allows to enable a mirror behavior on the old allocator to deal with problems during cluster upgrades. During the implementation the secondary range of the legacy allocator was initialized with the valuye of the primary range, hence, when a Service tried to allocate a new IP on the secondary range, it succeded in the new ip allocator but failed when it tried to allocate the same IP on the legacy allocator, since it has a different range. Expand the integration test that run over all the combinations of Service ClusterIP possibilities to run with all the possible combinations of the feature gates. The integration test need to change the way of starting the apiserver otherwise it will timeout. --- pkg/registry/core/rest/storage_core.go | 2 +- test/integration/dualstack/dualstack_test.go | 2069 +++++++++--------- 2 files changed, 1096 insertions(+), 975 deletions(-) diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index e5ced412aad..bac4b133de1 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -440,7 +440,7 @@ func (c *Config) newServiceIPAllocators() (registries rangeRegistries, primaryCl if err != nil { return nil, err } - rangeRegistry.Range = serviceClusterIPRange.String() + rangeRegistry.Range = c.Services.SecondaryClusterIPRange.String() if len(rangeRegistry.ResourceVersion) == 0 { klog.Infof("kube-apiserver started with IP allocator and dual write enabled but bitmap allocator does not exist, recreating it ...") err := etcd.CreateOrUpdate(rangeRegistry) diff --git a/test/integration/dualstack/dualstack_test.go b/test/integration/dualstack/dualstack_test.go index 01ab3323cd9..d6a1cc16727 100644 --- a/test/integration/dualstack/dualstack_test.go +++ b/test/integration/dualstack/dualstack_test.go @@ -31,11 +31,11 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" - - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/client-go/kubernetes" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/integration/framework" "k8s.io/kubernetes/test/utils/ktesting" netutils "k8s.io/utils/net" @@ -43,945 +43,1012 @@ import ( // TestCreateServiceSingleStackIPv4 test the Service dualstackness in an IPv4 SingleStack cluster func TestCreateServiceSingleStackIPv4(t *testing.T) { - // Create an IPv4 single stack control-plane - serviceCIDR := "10.0.0.0/16" - - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = serviceCIDR - }, - }) - defer tearDownFn() - - // Wait until the default "kubernetes" service is created. - if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - return !apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("creating kubernetes service timed out") - } - - var testcases = []struct { - name string - serviceType v1.ServiceType - clusterIPs []string - ipFamilies []v1.IPFamily - ipFamilyPolicy v1.IPFamilyPolicy - expectedIPFamilies []v1.IPFamily - expectError bool - }{ - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: nil, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.16"}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: nil, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - } - - for i, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test - }, - Spec: v1.ServiceSpec{ - Type: tc.serviceType, - ClusterIPs: tc.clusterIPs, - IPFamilies: tc.ipFamilies, - Ports: []v1.ServicePort{ - { - Port: 443, - TargetPort: intstr.FromInt32(443), - }, + for _, enableMultiServiceCIDR := range []bool{false, true} { + for _, disableAllocatorDualWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("MultiServiceCIDR=%v DisableAllocatorDualWrite=%v", enableMultiServiceCIDR, disableAllocatorDualWrite), func(t *testing.T) { + // Create an IPv4 single stack control-plane + tCtx := ktesting.Init(t) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + fmt.Sprintf("--runtime-config=networking.k8s.io/v1beta1=%v", enableMultiServiceCIDR), + "--service-cluster-ip-range=10.0.0.0/16", + "--advertise-address=10.1.1.1", + "--disable-admission-plugins=ServiceAccount", + fmt.Sprintf("--feature-gates=%s=%v,%s=%v", features.MultiCIDRServiceAllocator, enableMultiServiceCIDR, features.DisableAllocatorDualWrite, disableAllocatorDualWrite), }, - }, - } + etcdOptions) + defer s.TearDownFn() - if len(tc.ipFamilyPolicy) > 0 { - svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy - } + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } - if len(tc.clusterIPs) > 0 { - svc.Spec.ClusterIP = tc.clusterIPs[0] - } + // Wait until the default "kubernetes" service is created. + if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } - // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) - if (err != nil) != tc.expectError { - t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) - } - // if no error was expected validate the service otherwise return - if err != nil { - return - } - // validate the service was created correctly if it was not expected to fail - svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) - } - if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { - t.Errorf("Unexpected error validating the service %s\n%+v\n%v", svc.Name, svc, err) - } - }) + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicy + expectedIPFamilies []v1.IPFamily + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.16"}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt32(443), + }, + }, + }, + } + + if len(tc.ipFamilyPolicy) > 0 { + svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy + } + + if len(tc.clusterIPs) > 0 { + svc.Spec.ClusterIP = tc.clusterIPs[0] + } + + // create the service + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s\n%+v\n%v", svc.Name, svc, err) + } + }) + } + }) + } } } -// TestCreateServiceDualStackIPv6 test the Service dualstackness in an IPv6 only DualStack cluster -func TestCreateServiceDualStackIPv6(t *testing.T) { - // Create an IPv6 only dual stack control-plane - serviceCIDR := "2001:db8:1::/112" - - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = serviceCIDR - opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10") - }, - }) - defer tearDownFn() - - // Wait until the default "kubernetes" service is created. - if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - return !apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("creating kubernetes service timed out") - } - - var testcases = []struct { - name string - serviceType v1.ServiceType - clusterIPs []string - ipFamilies []v1.IPFamily - expectedIPFamilies []v1.IPFamily - ipFamilyPolicy v1.IPFamilyPolicy - expectError bool - }{ - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - } - - for i, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test - }, - Spec: v1.ServiceSpec{ - Type: tc.serviceType, - ClusterIPs: tc.clusterIPs, - IPFamilies: tc.ipFamilies, - IPFamilyPolicy: &tc.ipFamilyPolicy, - Ports: []v1.ServicePort{ - { - Name: fmt.Sprintf("port-test-%d", i), - Port: 443, - TargetPort: intstr.IntOrString{IntVal: 443}, - Protocol: "TCP", - }, +// TestCreateServiceSingleStackIPv6 test the Service dualstackness in an IPv6 only DualStack cluster +func TestCreateServiceSingleStackIPv6(t *testing.T) { + for _, enableMultiServiceCIDR := range []bool{false, true} { + for _, disableAllocatorDualWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("MultiServiceCIDR=%v DisableAllocatorDualWrite=%v", enableMultiServiceCIDR, disableAllocatorDualWrite), func(t *testing.T) { + // Create an IPv6 only control-plane + tCtx := ktesting.Init(t) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + fmt.Sprintf("--runtime-config=networking.k8s.io/v1beta1=%v", enableMultiServiceCIDR), + "--service-cluster-ip-range=2001:db8:1::/112", + "--advertise-address=2001:db8::10", + "--disable-admission-plugins=ServiceAccount", + fmt.Sprintf("--feature-gates=%s=%v,%s=%v", features.MultiCIDRServiceAllocator, enableMultiServiceCIDR, features.DisableAllocatorDualWrite, disableAllocatorDualWrite), }, - }, - } + etcdOptions) + defer s.TearDownFn() - // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) - if (err != nil) != tc.expectError { - t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) - } - // if no error was expected validate the service otherwise return - if err != nil { - return - } - // validate the service was created correctly if it was not expected to fail - svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) - } - if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { - t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) - } - }) + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // Wait until the default "kubernetes" service is created. + if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicy + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + IPFamilyPolicy: &tc.ipFamilyPolicy, + Ports: []v1.ServicePort{ + { + Name: fmt.Sprintf("port-test-%d", i), + Port: 443, + TargetPort: intstr.IntOrString{IntVal: 443}, + Protocol: "TCP", + }, + }, + }, + } + + // create the service + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } + }) + } } } // TestCreateServiceDualStackIPv4IPv6 test the Service dualstackness in a IPv4IPv6 DualStack cluster func TestCreateServiceDualStackIPv4IPv6(t *testing.T) { - // Create an IPv4IPv6 dual stack control-plane - serviceCIDR := "10.0.0.0/16" - secondaryServiceCIDR := "2001:db8:1::/112" - - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) - }, - }) - defer tearDownFn() - - // Wait until the default "kubernetes" service is created. - if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - return !apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("creating kubernetes service timed out") - } - - var testcases = []struct { - name string - serviceType v1.ServiceType - clusterIPs []string - ipFamilies []v1.IPFamily - expectedIPFamilies []v1.IPFamily - ipFamilyPolicy v1.IPFamilyPolicy - expectError bool - }{ - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv4 Family", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.16"}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv6 Family", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"2001:db8:1::16"}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.17", "2001:db8:1::17"}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.17", "2001:db8:1::17"}, - ipFamilies: nil, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.18", "2001:db8:1::18"}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - } - - for i, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test - }, - Spec: v1.ServiceSpec{ - Type: tc.serviceType, - ClusterIPs: tc.clusterIPs, - IPFamilies: tc.ipFamilies, - Ports: []v1.ServicePort{ - { - Port: 443, - TargetPort: intstr.FromInt32(443), - }, + for _, enableMultiServiceCIDR := range []bool{false, true} { + for _, disableAllocatorDualWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("MultiServiceCIDR=%v DisableAllocatorDualWrite=%v", enableMultiServiceCIDR, disableAllocatorDualWrite), func(t *testing.T) { + // Create an IPv4IPv6 dual stack control-plane + tCtx := ktesting.Init(t) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + fmt.Sprintf("--runtime-config=networking.k8s.io/v1beta1=%v", enableMultiServiceCIDR), + "--service-cluster-ip-range=10.0.0.0/16,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", + fmt.Sprintf("--feature-gates=%s=%v,%s=%v", features.MultiCIDRServiceAllocator, enableMultiServiceCIDR, features.DisableAllocatorDualWrite, disableAllocatorDualWrite), }, - }, - } + etcdOptions) + defer s.TearDownFn() - if len(tc.ipFamilyPolicy) > 0 { - svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy - } + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } - if len(tc.clusterIPs) > 0 { - svc.Spec.ClusterIP = tc.clusterIPs[0] - } + // Wait until the default "kubernetes" service is created. + if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } - // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) - if (err != nil) != tc.expectError { - t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) - } - // if no error was expected validate the service otherwise return - if err != nil { - return - } - // validate the service was created correctly if it was not expected to fail - svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) - } + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicy + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv4 Family", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.16"}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv6 Family", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"2001:db8:1::16"}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.17", "2001:db8:1::17"}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.17", "2001:db8:1::17"}, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.18", "2001:db8:1::18"}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, - if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { - t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) - } - }) + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt32(443), + }, + }, + }, + } + + if len(tc.ipFamilyPolicy) > 0 { + svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy + } + + if len(tc.clusterIPs) > 0 { + svc.Spec.ClusterIP = tc.clusterIPs[0] + } + + // create a service + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } + }) + } } } // TestCreateServiceDualStackIPv6IPv4 test the Service dualstackness in a IPv6IPv4 DualStack cluster func TestCreateServiceDualStackIPv6IPv4(t *testing.T) { - // Create an IPv6IPv4 dual stack control-plane - serviceCIDR := "2001:db8:1::/112" - secondaryServiceCIDR := "10.0.0.0/16" - - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) - opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10") - }, - }) - defer tearDownFn() - - // Wait until the default "kubernetes" service is created. - if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - return !apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("creating kubernetes service timed out") - } - - // verify client is working - if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { - _, err := client.CoreV1().Endpoints("default").Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil { - t.Logf("error fetching endpoints: %v", err) - return false, nil - } - return true, nil - }); err != nil { - t.Errorf("server without enabled endpoints failed to register: %v", err) - } - - var testcases = []struct { - name string - serviceType v1.ServiceType - clusterIPs []string - ipFamilies []v1.IPFamily - expectedIPFamilies []v1.IPFamily - ipFamilyPolicy v1.IPFamilyPolicy - expectError bool - }{ - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - } - - for i, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test - }, - Spec: v1.ServiceSpec{ - Type: tc.serviceType, - ClusterIPs: tc.clusterIPs, - IPFamilies: tc.ipFamilies, - IPFamilyPolicy: &tc.ipFamilyPolicy, - Ports: []v1.ServicePort{ - { - Port: 443, - TargetPort: intstr.FromInt32(443), - }, + for _, enableMultiServiceCIDR := range []bool{false, true} { + for _, disableAllocatorDualWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("MultiServiceCIDR=%v DisableAllocatorDualWrite=%v", enableMultiServiceCIDR, disableAllocatorDualWrite), func(t *testing.T) { + // Create an IPv6IPv4 dual stack control-plane + tCtx := ktesting.Init(t) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + fmt.Sprintf("--runtime-config=networking.k8s.io/v1beta1=%v", enableMultiServiceCIDR), + "--service-cluster-ip-range=2001:db8:1::/112,10.0.0.0/16", + "--advertise-address=2001:db8::10", + "--disable-admission-plugins=ServiceAccount", + fmt.Sprintf("--feature-gates=%s=%v,%s=%v", features.MultiCIDRServiceAllocator, enableMultiServiceCIDR, features.DisableAllocatorDualWrite, disableAllocatorDualWrite), }, - }, - } + etcdOptions) + defer s.TearDownFn() - // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) - if (err != nil) != tc.expectError { - t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) - } - // if no error was expected validate the service otherwise return - if err != nil { - return - } - // validate the service was created correctly if it was not expected to fail - svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) - } + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } - if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { - t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) - } - }) + // Wait until the default "kubernetes" service is created. + if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + // verify client is working + if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { + _, err := client.CoreV1().Endpoints("default").Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil { + t.Logf("error fetching endpoints: %v", err) + return false, nil + } + return true, nil + }); err != nil { + t.Errorf("server without enabled endpoints failed to register: %v", err) + } + + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicy + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + IPFamilyPolicy: &tc.ipFamilyPolicy, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt32(443), + }, + }, + }, + } + + // create a service + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } + }) + } } } // TestUpgradeDowngrade tests upgrading and downgrading a service from/to dual-stack func TestUpgradeDowngrade(t *testing.T) { // Create an IPv4IPv6 dual stack control-plane - serviceCIDR := "10.0.0.0/16" - secondaryServiceCIDR := "2001:db8:1::/112" - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=10.0.0.0/16,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1012,7 +1079,7 @@ func TestUpgradeDowngrade(t *testing.T) { } // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("unexpected error while creating service:%v", err) } @@ -1076,16 +1143,23 @@ func TestUpgradeDowngrade(t *testing.T) { // may not clear ClusterIPs func TestConvertToFromExternalName(t *testing.T) { // Create an IPv4IPv6 dual stack control-plane - serviceCIDR := "10.0.0.0/16" - secondaryServiceCIDR := "2001:db8:1::/112" - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=10.0.0.0/16,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1115,7 +1189,7 @@ func TestConvertToFromExternalName(t *testing.T) { } // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("unexpected error while creating service:%v", err) } @@ -1158,16 +1232,23 @@ func TestConvertToFromExternalName(t *testing.T) { // TestPreferDualStack preferDualstack on create and update func TestPreferDualStack(t *testing.T) { // Create an IPv4IPv6 dual stack control-plane - serviceCIDR := "10.0.0.0/16" - secondaryServiceCIDR := "2001:db8:1::/112" - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=10.0.0.0/16,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1201,7 +1282,7 @@ func TestPreferDualStack(t *testing.T) { } // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("unexpected error while creating service:%v", err) } @@ -1233,15 +1314,23 @@ type labelsForMergePatch struct { // tests an update service while dualstack flag is off func TestServiceUpdate(t *testing.T) { // Create an IPv4 single stack control-plane - serviceCIDR := "10.0.0.0/16" - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = serviceCIDR + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=10.0.0.0/16", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1271,7 +1360,7 @@ func TestServiceUpdate(t *testing.T) { } // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) // if no error was expected validate the service otherwise return if err != nil { t.Errorf("unexpected error creating service:%v", err) @@ -1393,18 +1482,23 @@ func validateServiceAndClusterIPFamily(svc *v1.Service, expectedIPFamilies []v1. func TestUpgradeServicePreferToDualStack(t *testing.T) { sharedEtcd := framework.SharedEtcd() - tCtx := ktesting.Init(t) // Create an IPv4 only dual stack control-plane - serviceCIDR := "192.168.0.0/24" - - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.Etcd.StorageConfig = *sharedEtcd - opts.ServiceClusterIPRanges = serviceCIDR + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) + sharedEtcd) + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1439,7 +1533,7 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) { } // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -1453,17 +1547,21 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) { } // reconfigure the apiserver to be dual-stack - tearDownFn() + s.TearDownFn() - secondaryServiceCIDR := "2001:db8:1::/112" - - client, _, tearDownFn = framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.Etcd.StorageConfig = *sharedEtcd - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + s = kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24,2001:db8:1::/112", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + sharedEtcd) + defer s.TearDownFn() + + client, err = kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1487,20 +1585,24 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) { } func TestDowngradeServicePreferToDualStack(t *testing.T) { - sharedEtcd := framework.SharedEtcd() - tCtx := ktesting.Init(t) // Create a dual stack control-plane - serviceCIDR := "192.168.0.0/24" - secondaryServiceCIDR := "2001:db8:1::/112" - - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.Etcd.StorageConfig = *sharedEtcd - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) + etcdOptions) + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1533,7 +1635,7 @@ func TestDowngradeServicePreferToDualStack(t *testing.T) { }, } // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -1545,18 +1647,24 @@ func TestDowngradeServicePreferToDualStack(t *testing.T) { if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil { t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err) } - // reconfigure the apiserver to be sinlge stack - tearDownFn() + // reconfigure the apiserver to be single stack + s.TearDownFn() // reset secondary - client, _, tearDownFn = framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.Etcd.StorageConfig = *sharedEtcd - opts.ServiceClusterIPRanges = serviceCIDR + s = kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + client, err = kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) @@ -1589,8 +1697,22 @@ type specMergePatch struct { // tests success when converting ClusterIP:Headless service to ExternalName func Test_ServiceChangeTypeHeadlessToExternalNameWithPatch(t *testing.T) { tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{}) - defer tearDownFn() + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", + }, + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } ns := framework.CreateNamespaceOrDie(client, "test-service-allocate-node-ports", t) defer framework.DeleteNamespaceOrDie(client, ns, t) @@ -1606,7 +1728,6 @@ func Test_ServiceChangeTypeHeadlessToExternalNameWithPatch(t *testing.T) { }, } - var err error service, err = client.CoreV1().Services(ns.Name).Create(tCtx, service, metav1.CreateOptions{}) if err != nil { t.Fatalf("Error creating test service: %v", err)