diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index e5ced412aad..bac4b133de1 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -440,7 +440,7 @@ func (c *Config) newServiceIPAllocators() (registries rangeRegistries, primaryCl if err != nil { return nil, err } - rangeRegistry.Range = serviceClusterIPRange.String() + rangeRegistry.Range = c.Services.SecondaryClusterIPRange.String() if len(rangeRegistry.ResourceVersion) == 0 { klog.Infof("kube-apiserver started with IP allocator and dual write enabled but bitmap allocator does not exist, recreating it ...") err := etcd.CreateOrUpdate(rangeRegistry) diff --git a/test/integration/dualstack/dualstack_test.go b/test/integration/dualstack/dualstack_test.go index 01ab3323cd9..d6a1cc16727 100644 --- a/test/integration/dualstack/dualstack_test.go +++ b/test/integration/dualstack/dualstack_test.go @@ -31,11 +31,11 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" - - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/client-go/kubernetes" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/integration/framework" "k8s.io/kubernetes/test/utils/ktesting" netutils "k8s.io/utils/net" @@ -43,945 +43,1012 @@ import ( // TestCreateServiceSingleStackIPv4 test the Service dualstackness in an IPv4 SingleStack cluster func TestCreateServiceSingleStackIPv4(t *testing.T) { - // Create an IPv4 single stack control-plane - serviceCIDR := "10.0.0.0/16" - - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = serviceCIDR - }, - }) - defer tearDownFn() - - // Wait until the default "kubernetes" service is created. - if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - return !apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("creating kubernetes service timed out") - } - - var testcases = []struct { - name string - serviceType v1.ServiceType - clusterIPs []string - ipFamilies []v1.IPFamily - ipFamilyPolicy v1.IPFamilyPolicy - expectedIPFamilies []v1.IPFamily - expectError bool - }{ - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: nil, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.16"}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: nil, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: nil, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: true, - }, - } - - for i, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test - }, - Spec: v1.ServiceSpec{ - Type: tc.serviceType, - ClusterIPs: tc.clusterIPs, - IPFamilies: tc.ipFamilies, - Ports: []v1.ServicePort{ - { - Port: 443, - TargetPort: intstr.FromInt32(443), - }, + for _, enableMultiServiceCIDR := range []bool{false, true} { + for _, disableAllocatorDualWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("MultiServiceCIDR=%v DisableAllocatorDualWrite=%v", enableMultiServiceCIDR, disableAllocatorDualWrite), func(t *testing.T) { + // Create an IPv4 single stack control-plane + tCtx := ktesting.Init(t) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + fmt.Sprintf("--runtime-config=networking.k8s.io/v1beta1=%v", enableMultiServiceCIDR), + "--service-cluster-ip-range=10.0.0.0/16", + "--advertise-address=10.1.1.1", + "--disable-admission-plugins=ServiceAccount", + fmt.Sprintf("--feature-gates=%s=%v,%s=%v", features.MultiCIDRServiceAllocator, enableMultiServiceCIDR, features.DisableAllocatorDualWrite, disableAllocatorDualWrite), }, - }, - } + etcdOptions) + defer s.TearDownFn() - if len(tc.ipFamilyPolicy) > 0 { - svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy - } + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } - if len(tc.clusterIPs) > 0 { - svc.Spec.ClusterIP = tc.clusterIPs[0] - } + // Wait until the default "kubernetes" service is created. + if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } - // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) - if (err != nil) != tc.expectError { - t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) - } - // if no error was expected validate the service otherwise return - if err != nil { - return - } - // validate the service was created correctly if it was not expected to fail - svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) - } - if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { - t.Errorf("Unexpected error validating the service %s\n%+v\n%v", svc.Name, svc, err) - } - }) + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicy + expectedIPFamilies []v1.IPFamily + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.16"}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt32(443), + }, + }, + }, + } + + if len(tc.ipFamilyPolicy) > 0 { + svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy + } + + if len(tc.clusterIPs) > 0 { + svc.Spec.ClusterIP = tc.clusterIPs[0] + } + + // create the service + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s\n%+v\n%v", svc.Name, svc, err) + } + }) + } + }) + } } } -// TestCreateServiceDualStackIPv6 test the Service dualstackness in an IPv6 only DualStack cluster -func TestCreateServiceDualStackIPv6(t *testing.T) { - // Create an IPv6 only dual stack control-plane - serviceCIDR := "2001:db8:1::/112" - - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = serviceCIDR - opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10") - }, - }) - defer tearDownFn() - - // Wait until the default "kubernetes" service is created. - if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - return !apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("creating kubernetes service timed out") - } - - var testcases = []struct { - name string - serviceType v1.ServiceType - clusterIPs []string - ipFamilies []v1.IPFamily - expectedIPFamilies []v1.IPFamily - ipFamilyPolicy v1.IPFamilyPolicy - expectError bool - }{ - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: true, - }, - } - - for i, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test - }, - Spec: v1.ServiceSpec{ - Type: tc.serviceType, - ClusterIPs: tc.clusterIPs, - IPFamilies: tc.ipFamilies, - IPFamilyPolicy: &tc.ipFamilyPolicy, - Ports: []v1.ServicePort{ - { - Name: fmt.Sprintf("port-test-%d", i), - Port: 443, - TargetPort: intstr.IntOrString{IntVal: 443}, - Protocol: "TCP", - }, +// TestCreateServiceSingleStackIPv6 test the Service dualstackness in an IPv6 only DualStack cluster +func TestCreateServiceSingleStackIPv6(t *testing.T) { + for _, enableMultiServiceCIDR := range []bool{false, true} { + for _, disableAllocatorDualWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("MultiServiceCIDR=%v DisableAllocatorDualWrite=%v", enableMultiServiceCIDR, disableAllocatorDualWrite), func(t *testing.T) { + // Create an IPv6 only control-plane + tCtx := ktesting.Init(t) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + fmt.Sprintf("--runtime-config=networking.k8s.io/v1beta1=%v", enableMultiServiceCIDR), + "--service-cluster-ip-range=2001:db8:1::/112", + "--advertise-address=2001:db8::10", + "--disable-admission-plugins=ServiceAccount", + fmt.Sprintf("--feature-gates=%s=%v,%s=%v", features.MultiCIDRServiceAllocator, enableMultiServiceCIDR, features.DisableAllocatorDualWrite, disableAllocatorDualWrite), }, - }, - } + etcdOptions) + defer s.TearDownFn() - // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) - if (err != nil) != tc.expectError { - t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) - } - // if no error was expected validate the service otherwise return - if err != nil { - return - } - // validate the service was created correctly if it was not expected to fail - svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) - } - if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { - t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) - } - }) + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // Wait until the default "kubernetes" service is created. + if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicy + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + IPFamilyPolicy: &tc.ipFamilyPolicy, + Ports: []v1.ServicePort{ + { + Name: fmt.Sprintf("port-test-%d", i), + Port: 443, + TargetPort: intstr.IntOrString{IntVal: 443}, + Protocol: "TCP", + }, + }, + }, + } + + // create the service + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } + }) + } } } // TestCreateServiceDualStackIPv4IPv6 test the Service dualstackness in a IPv4IPv6 DualStack cluster func TestCreateServiceDualStackIPv4IPv6(t *testing.T) { - // Create an IPv4IPv6 dual stack control-plane - serviceCIDR := "10.0.0.0/16" - secondaryServiceCIDR := "2001:db8:1::/112" - - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) - }, - }) - defer tearDownFn() - - // Wait until the default "kubernetes" service is created. - if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - return !apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("creating kubernetes service timed out") - } - - var testcases = []struct { - name string - serviceType v1.ServiceType - clusterIPs []string - ipFamilies []v1.IPFamily - expectedIPFamilies []v1.IPFamily - ipFamilyPolicy v1.IPFamilyPolicy - expectError bool - }{ - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv4 Family", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.16"}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv6 Family", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"2001:db8:1::16"}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.17", "2001:db8:1::17"}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.17", "2001:db8:1::17"}, - ipFamilies: nil, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectError: false, - }, - { - name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{"10.0.0.18", "2001:db8:1::18"}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - } - - for i, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test - }, - Spec: v1.ServiceSpec{ - Type: tc.serviceType, - ClusterIPs: tc.clusterIPs, - IPFamilies: tc.ipFamilies, - Ports: []v1.ServicePort{ - { - Port: 443, - TargetPort: intstr.FromInt32(443), - }, + for _, enableMultiServiceCIDR := range []bool{false, true} { + for _, disableAllocatorDualWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("MultiServiceCIDR=%v DisableAllocatorDualWrite=%v", enableMultiServiceCIDR, disableAllocatorDualWrite), func(t *testing.T) { + // Create an IPv4IPv6 dual stack control-plane + tCtx := ktesting.Init(t) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + fmt.Sprintf("--runtime-config=networking.k8s.io/v1beta1=%v", enableMultiServiceCIDR), + "--service-cluster-ip-range=10.0.0.0/16,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", + fmt.Sprintf("--feature-gates=%s=%v,%s=%v", features.MultiCIDRServiceAllocator, enableMultiServiceCIDR, features.DisableAllocatorDualWrite, disableAllocatorDualWrite), }, - }, - } + etcdOptions) + defer s.TearDownFn() - if len(tc.ipFamilyPolicy) > 0 { - svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy - } + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } - if len(tc.clusterIPs) > 0 { - svc.Spec.ClusterIP = tc.clusterIPs[0] - } + // Wait until the default "kubernetes" service is created. + if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } - // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) - if (err != nil) != tc.expectError { - t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) - } - // if no error was expected validate the service otherwise return - if err != nil { - return - } - // validate the service was created correctly if it was not expected to fail - svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) - } + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicy + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv4 Family", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.16"}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv6 Family", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"2001:db8:1::16"}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.17", "2001:db8:1::17"}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.17", "2001:db8:1::17"}, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{"10.0.0.18", "2001:db8:1::18"}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, - if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { - t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) - } - }) + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt32(443), + }, + }, + }, + } + + if len(tc.ipFamilyPolicy) > 0 { + svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy + } + + if len(tc.clusterIPs) > 0 { + svc.Spec.ClusterIP = tc.clusterIPs[0] + } + + // create a service + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } + }) + } } } // TestCreateServiceDualStackIPv6IPv4 test the Service dualstackness in a IPv6IPv4 DualStack cluster func TestCreateServiceDualStackIPv6IPv4(t *testing.T) { - // Create an IPv6IPv4 dual stack control-plane - serviceCIDR := "2001:db8:1::/112" - secondaryServiceCIDR := "10.0.0.0/16" - - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) - opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10") - }, - }) - defer tearDownFn() - - // Wait until the default "kubernetes" service is created. - if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - return !apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("creating kubernetes service timed out") - } - - // verify client is working - if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { - _, err := client.CoreV1().Endpoints("default").Get(tCtx, "kubernetes", metav1.GetOptions{}) - if err != nil { - t.Logf("error fetching endpoints: %v", err) - return false, nil - } - return true, nil - }); err != nil { - t.Errorf("server without enabled endpoints failed to register: %v", err) - } - - var testcases = []struct { - name string - serviceType v1.ServiceType - clusterIPs []string - ipFamilies []v1.IPFamily - expectedIPFamilies []v1.IPFamily - ipFamilyPolicy v1.IPFamilyPolicy - expectError bool - }{ - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: nil, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicySingleStack, - expectError: true, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, - expectError: false, - }, - { - name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", - serviceType: v1.ServiceTypeClusterIP, - clusterIPs: []string{}, - ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, - ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, - expectError: false, - }, - } - - for i, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test - }, - Spec: v1.ServiceSpec{ - Type: tc.serviceType, - ClusterIPs: tc.clusterIPs, - IPFamilies: tc.ipFamilies, - IPFamilyPolicy: &tc.ipFamilyPolicy, - Ports: []v1.ServicePort{ - { - Port: 443, - TargetPort: intstr.FromInt32(443), - }, + for _, enableMultiServiceCIDR := range []bool{false, true} { + for _, disableAllocatorDualWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("MultiServiceCIDR=%v DisableAllocatorDualWrite=%v", enableMultiServiceCIDR, disableAllocatorDualWrite), func(t *testing.T) { + // Create an IPv6IPv4 dual stack control-plane + tCtx := ktesting.Init(t) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + fmt.Sprintf("--runtime-config=networking.k8s.io/v1beta1=%v", enableMultiServiceCIDR), + "--service-cluster-ip-range=2001:db8:1::/112,10.0.0.0/16", + "--advertise-address=2001:db8::10", + "--disable-admission-plugins=ServiceAccount", + fmt.Sprintf("--feature-gates=%s=%v,%s=%v", features.MultiCIDRServiceAllocator, enableMultiServiceCIDR, features.DisableAllocatorDualWrite, disableAllocatorDualWrite), }, - }, - } + etcdOptions) + defer s.TearDownFn() - // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) - if (err != nil) != tc.expectError { - t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) - } - // if no error was expected validate the service otherwise return - if err != nil { - return - } - // validate the service was created correctly if it was not expected to fail - svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) - } + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } - if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { - t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) - } - }) + // Wait until the default "kubernetes" service is created. + if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + // verify client is working + if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { + _, err := client.CoreV1().Endpoints("default").Get(tCtx, "kubernetes", metav1.GetOptions{}) + if err != nil { + t.Logf("error fetching endpoints: %v", err) + return false, nil + } + return true, nil + }); err != nil { + t.Errorf("server without enabled endpoints failed to register: %v", err) + } + + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicy + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + IPFamilyPolicy: &tc.ipFamilyPolicy, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt32(443), + }, + }, + }, + } + + // create a service + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } + }) + } } } // TestUpgradeDowngrade tests upgrading and downgrading a service from/to dual-stack func TestUpgradeDowngrade(t *testing.T) { // Create an IPv4IPv6 dual stack control-plane - serviceCIDR := "10.0.0.0/16" - secondaryServiceCIDR := "2001:db8:1::/112" - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=10.0.0.0/16,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1012,7 +1079,7 @@ func TestUpgradeDowngrade(t *testing.T) { } // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("unexpected error while creating service:%v", err) } @@ -1076,16 +1143,23 @@ func TestUpgradeDowngrade(t *testing.T) { // may not clear ClusterIPs func TestConvertToFromExternalName(t *testing.T) { // Create an IPv4IPv6 dual stack control-plane - serviceCIDR := "10.0.0.0/16" - secondaryServiceCIDR := "2001:db8:1::/112" - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=10.0.0.0/16,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1115,7 +1189,7 @@ func TestConvertToFromExternalName(t *testing.T) { } // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("unexpected error while creating service:%v", err) } @@ -1158,16 +1232,23 @@ func TestConvertToFromExternalName(t *testing.T) { // TestPreferDualStack preferDualstack on create and update func TestPreferDualStack(t *testing.T) { // Create an IPv4IPv6 dual stack control-plane - serviceCIDR := "10.0.0.0/16" - secondaryServiceCIDR := "2001:db8:1::/112" - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=10.0.0.0/16,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1201,7 +1282,7 @@ func TestPreferDualStack(t *testing.T) { } // create a service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("unexpected error while creating service:%v", err) } @@ -1233,15 +1314,23 @@ type labelsForMergePatch struct { // tests an update service while dualstack flag is off func TestServiceUpdate(t *testing.T) { // Create an IPv4 single stack control-plane - serviceCIDR := "10.0.0.0/16" - tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.ServiceClusterIPRanges = serviceCIDR + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=10.0.0.0/16", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1271,7 +1360,7 @@ func TestServiceUpdate(t *testing.T) { } // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) // if no error was expected validate the service otherwise return if err != nil { t.Errorf("unexpected error creating service:%v", err) @@ -1393,18 +1482,23 @@ func validateServiceAndClusterIPFamily(svc *v1.Service, expectedIPFamilies []v1. func TestUpgradeServicePreferToDualStack(t *testing.T) { sharedEtcd := framework.SharedEtcd() - tCtx := ktesting.Init(t) // Create an IPv4 only dual stack control-plane - serviceCIDR := "192.168.0.0/24" - - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.Etcd.StorageConfig = *sharedEtcd - opts.ServiceClusterIPRanges = serviceCIDR + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) + sharedEtcd) + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1439,7 +1533,7 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) { } // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -1453,17 +1547,21 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) { } // reconfigure the apiserver to be dual-stack - tearDownFn() + s.TearDownFn() - secondaryServiceCIDR := "2001:db8:1::/112" - - client, _, tearDownFn = framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.Etcd.StorageConfig = *sharedEtcd - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + s = kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24,2001:db8:1::/112", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + sharedEtcd) + defer s.TearDownFn() + + client, err = kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1487,20 +1585,24 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) { } func TestDowngradeServicePreferToDualStack(t *testing.T) { - sharedEtcd := framework.SharedEtcd() - tCtx := ktesting.Init(t) // Create a dual stack control-plane - serviceCIDR := "192.168.0.0/24" - secondaryServiceCIDR := "2001:db8:1::/112" - - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.Etcd.StorageConfig = *sharedEtcd - opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24,2001:db8:1::/112", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) + etcdOptions) + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { @@ -1533,7 +1635,7 @@ func TestDowngradeServicePreferToDualStack(t *testing.T) { }, } // create the service - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -1545,18 +1647,24 @@ func TestDowngradeServicePreferToDualStack(t *testing.T) { if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil { t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err) } - // reconfigure the apiserver to be sinlge stack - tearDownFn() + // reconfigure the apiserver to be single stack + s.TearDownFn() // reset secondary - client, _, tearDownFn = framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - opts.Etcd.StorageConfig = *sharedEtcd - opts.ServiceClusterIPRanges = serviceCIDR + s = kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", }, - }) - defer tearDownFn() + etcdOptions) + defer s.TearDownFn() + client, err = kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } // Wait until the default "kubernetes" service is created. if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{}) @@ -1589,8 +1697,22 @@ type specMergePatch struct { // tests success when converting ClusterIP:Headless service to ExternalName func Test_ServiceChangeTypeHeadlessToExternalNameWithPatch(t *testing.T) { tCtx := ktesting.Init(t) - client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{}) - defer tearDownFn() + etcdOptions := framework.SharedEtcd() + apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() + s := kubeapiservertesting.StartTestServerOrDie(t, + apiServerOptions, + []string{ + "--service-cluster-ip-range=192.168.0.0/24", + "--advertise-address=10.0.0.1", + "--disable-admission-plugins=ServiceAccount", + }, + etcdOptions) + defer s.TearDownFn() + + client, err := kubernetes.NewForConfig(s.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } ns := framework.CreateNamespaceOrDie(client, "test-service-allocate-node-ports", t) defer framework.DeleteNamespaceOrDie(client, ns, t) @@ -1606,7 +1728,6 @@ func Test_ServiceChangeTypeHeadlessToExternalNameWithPatch(t *testing.T) { }, } - var err error service, err = client.CoreV1().Services(ns.Name).Create(tCtx, service, metav1.CreateOptions{}) if err != nil { t.Fatalf("Error creating test service: %v", err)